diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
index 039a51072be..078a8c054d1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
@@ -46,6 +46,7 @@
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.DelegatingLinuxContainerRuntime;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.DockerLinuxContainerRuntime;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntime;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.OCIContainerRuntime;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker.DockerCommandExecutor;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker.DockerRmCommand;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer;
@@ -94,14 +95,14 @@
* appropriate {@link LinuxContainerRuntime} instance. This class uses a
* {@link DelegatingLinuxContainerRuntime} instance, which will delegate calls
* to either a {@link DefaultLinuxContainerRuntime} instance or a
- * {@link DockerLinuxContainerRuntime} instance, depending on the job's
+ * {@link OCIContainerRuntime} instance, depending on the job's
* configuration.
*
* @see LinuxContainerRuntime
* @see DelegatingLinuxContainerRuntime
* @see DefaultLinuxContainerRuntime
* @see DockerLinuxContainerRuntime
- * @see DockerLinuxContainerRuntime#isDockerContainerRequested
+ * @see OCIContainerRuntime#isOCICompliantContainerRequested
*/
public class LinuxContainerExecutor extends ContainerExecutor {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerCleanup.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerCleanup.java
index faf926aab64..e92560e3f87 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerCleanup.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerCleanup.java
@@ -35,7 +35,7 @@
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerEventType;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerExitEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.deletion.task.DockerContainerDeletionTask;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.DockerLinuxContainerRuntime;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.OCIContainerRuntime;
import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerReapContext;
import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerSignalContext;
import org.slf4j.Logger;
@@ -147,7 +147,7 @@ public void run() {
}
// rm container in docker
- if (DockerLinuxContainerRuntime.isDockerContainerRequested(conf,
+ if (OCIContainerRuntime.isOCICompliantContainerRequested(conf,
container.getLaunchContext().getEnvironment())) {
rmDockerContainerDelayed();
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceHandlerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceHandlerImpl.java
index 9474b0f8471..9811b22fd7c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceHandlerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceHandlerImpl.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.gpu;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.OCIContainerRuntime;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
@@ -32,7 +33,6 @@
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.CGroupsHandler;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandler;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerException;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.DockerLinuxContainerRuntime;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.gpu.GpuDevice;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.gpu.GpuDiscoverer;
@@ -105,7 +105,7 @@ public GpuResourceHandlerImpl(Context nmContext,
// Create device cgroups for the container
cGroupsHandler.createCGroup(CGroupsHandler.CGroupController.DEVICES,
containerIdStr);
- if (!DockerLinuxContainerRuntime.isDockerContainerRequested(
+ if (!OCIContainerRuntime.isOCICompliantContainerRequested(
nmContext.getConf(),
container.getLaunchContext().getEnvironment())) {
// Write to devices cgroup only for non-docker container. The reason is
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
index 85ddca90dad..98ed49cd41b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
@@ -23,9 +23,6 @@
import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.hdfs.protocol.datatransfer.IOStreamPair;
import org.apache.hadoop.security.Credentials;
-import org.apache.hadoop.yarn.api.ApplicationConstants.Environment;
-import org.apache.hadoop.yarn.api.CsiAdaptorProtocol;
-import org.apache.hadoop.yarn.api.impl.pb.client.CsiAdaptorProtocolPBClientImpl;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.nodemanager.Context;
@@ -41,7 +38,6 @@
import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.ResourcePlugin;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.volume.csi.ContainerVolumePublisher;
import org.apache.hadoop.yarn.util.DockerClientConfigHandler;
-import org.apache.hadoop.yarn.util.csi.CsiConfigUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.classification.InterfaceAudience;
@@ -50,9 +46,6 @@
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.registry.client.api.RegistryConstants;
import org.apache.hadoop.registry.client.binding.RegistryPathUtils;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.authorize.AccessControlList;
-import org.apache.hadoop.util.Shell;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
@@ -67,7 +60,6 @@
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker.DockerInspectCommand;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker.DockerRunCommand;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerExecutionException;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerRuntime;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerRuntimeConstants;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerRuntimeContext;
import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerExecContext;
@@ -75,19 +67,11 @@
import java.io.File;
import java.io.IOException;
import java.net.InetAddress;
-import java.net.InetSocketAddress;
import java.net.UnknownHostException;
import java.nio.ByteBuffer;
-import java.nio.file.Files;
-import java.nio.file.Paths;
import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
import java.util.List;
import java.util.Map;
-import java.util.Map.Entry;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -95,7 +79,7 @@
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.*;
/**
- * This class is a {@link ContainerRuntime} implementation that uses the
+ *
This class is an extension of {@link OCIContainerRuntime} that uses the
* native {@code container-executor} binary via a
* {@link PrivilegedOperationExecutor} instance to launch processes inside
* Docker containers.
@@ -107,22 +91,9 @@
*
* {@code YARN_CONTAINER_RUNTIME_TYPE} ultimately determines whether a
* Docker container will be used. If the value is {@code docker}, a Docker
- * container will be used. Otherwise a regular process tree container will
- * be used. This environment variable is checked by the
+ * container will be used. This environment variable is checked by the
* {@link #isDockerContainerRequested} method, which is called by the
* {@link DelegatingLinuxContainerRuntime}.
- *
- *
- * {@code YARN_CONTAINER_RUNTIME_DOCKER_IMAGE} names which image
- * will be used to launch the Docker container.
- *
- *
- * {@code YARN_CONTAINER_RUNTIME_DOCKER_RUN_OVERRIDE_DISABLE} controls
- * whether the Docker container's default command is overridden. When set
- * to {@code true}, the Docker container's command will be
- * {@code bash }. When unset or set to {@code false}
- * the Docker container's default command is used.
- *
*
* {@code YARN_CONTAINER_RUNTIME_DOCKER_CONTAINER_NETWORK} sets the
* network type to be used by the Docker container. It must be a valid
@@ -130,167 +101,21 @@
* {@code yarn.nodemanager.runtime.linux.docker.allowed-container-networks}
* property.
*
- *
- * {@code YARN_CONTAINER_RUNTIME_DOCKER_PORTS_MAPPING} allows users to
- * specify ports mapping for the bridge network Docker container. The value
- * of the environment variable should be a comma-separated list of ports
- * mapping. It's the same to "-p" option for the Docker run command. If the
- * value is empty, "-P" will be added.
- *
- *
- * {@code YARN_CONTAINER_RUNTIME_DOCKER_CONTAINER_PID_NAMESPACE}
- * controls which PID namespace will be used by the Docker container. By
- * default, each Docker container has its own PID namespace. To share the
- * namespace of the host, the
- * {@code yarn.nodemanager.runtime.linux.docker.host-pid-namespace.allowed}
- * property must be set to {@code true}. If the host PID namespace is
- * allowed and this environment variable is set to {@code host}, the
- * Docker container will share the host's PID namespace. No other value is
- * allowed.
- *
- *
- * {@code YARN_CONTAINER_RUNTIME_DOCKER_CONTAINER_HOSTNAME} sets the
- * hostname to be used by the Docker container. If not specified, a
- * hostname will be derived from the container ID and set as default
- * hostname for networks other than 'host'.
- *
- *
- * {@code YARN_CONTAINER_RUNTIME_DOCKER_RUN_PRIVILEGED_CONTAINER}
- * controls whether the Docker container is a privileged container. In order
- * to use privileged containers, the
- * {@code yarn.nodemanager.runtime.linux.docker.privileged-containers.allowed}
- * property must be set to {@code true}, and the application owner must
- * appear in the value of the
- * {@code yarn.nodemanager.runtime.linux.docker.privileged-containers.acl}
- * property. If this environment variable is set to {@code true}, a
- * privileged Docker container will be used if allowed. No other value is
- * allowed, so the environment variable should be left unset rather than
- * setting it to false.
- *
- *
- * {@code YARN_CONTAINER_RUNTIME_DOCKER_MOUNTS} allows users to specify
- + additional volume mounts for the Docker container. The value of the
- * environment variable should be a comma-separated list of mounts.
- * All such mounts must be given as {@code source:dest[:mode]} and the mode
- * must be "ro" (read-only) or "rw" (read-write) to specify the type of
- * access being requested. If neither is specified, read-write will be
- * assumed. The mode may include a bind propagation option. In that case,
- * the mode should either be of the form [option], rw+[option], or
- * ro+[option]. Valid bind propagation options are shared, rshared, slave,
- * rslave, private, and rprivate. The requested mounts will be validated by
- * container-executor based on the values set in container-executor.cfg for
- * {@code docker.allowed.ro-mounts} and {@code docker.allowed.rw-mounts}.
- *
- *
- * {@code YARN_CONTAINER_RUNTIME_DOCKER_TMPFS_MOUNTS} allows users to
- * specify additional tmpfs mounts for the Docker container. The value of
- * the environment variable should be a comma-separated list of mounts.
- *
- *
- * {@code YARN_CONTAINER_RUNTIME_DOCKER_DELAYED_REMOVAL} allows a user
- * to request delayed deletion of the Docker containers on a per
- * container basis. If true, Docker containers will not be removed until
- * the duration defined by {@code yarn.nodemanager.delete.debug-delay-sec}
- * has elapsed. Administrators can disable this feature through the
- * yarn-site property
- * {@code yarn.nodemanager.runtime.linux.docker.delayed-removal.allowed}.
- * This feature is disabled by default. When this feature is disabled or set
- * to false, the container will be removed as soon as it exits.
- *
- *
- * {@code YARN_CONTAINER_RUNTIME_YARN_SYSFS_ENABLE} allows export yarn
- * service json to docker container. This feature is disabled by default.
- * when this feature is set, app.json will be available in
- * /hadoop/yarn/sysfs/app.json.
- *
*
*/
@InterfaceAudience.Private
@InterfaceStability.Unstable
-public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
+public class DockerLinuxContainerRuntime extends OCIContainerRuntime {
private static final Logger LOG =
- LoggerFactory.getLogger(DockerLinuxContainerRuntime.class);
-
- // This validates that the image is a proper docker image
- public static final String DOCKER_IMAGE_PATTERN =
- "^(([a-zA-Z0-9.-]+)(:\\d+)?/)?([a-z0-9_./-]+)(:[\\w.-]+)?$";
- private static final Pattern dockerImagePattern =
- Pattern.compile(DOCKER_IMAGE_PATTERN);
- public static final String HOSTNAME_PATTERN =
- "^[a-zA-Z0-9][a-zA-Z0-9_.-]+$";
- private static final Pattern hostnamePattern = Pattern.compile(
- HOSTNAME_PATTERN);
- private static final Pattern USER_MOUNT_PATTERN = Pattern.compile(
- "(?<=^|,)([^:\\x00]+):([^:\\x00]+)" +
- "(:(r[ow]|(r[ow][+])?(r?shared|r?slave|r?private)))?(?:,|$)");
- private static final Pattern TMPFS_MOUNT_PATTERN = Pattern.compile(
- "^/[^:\\x00]+$");
- public static final String PORTS_MAPPING_PATTERN =
- "^:[0-9]+|^[0-9]+:[0-9]+|^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]" +
- "|25[0-5])\\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])" +
- ":[0-9]+:[0-9]+$";
- private static final int HOST_NAME_LENGTH = 64;
+ LoggerFactory.getLogger(DockerLinuxContainerRuntime.class);
+
private static final String DEFAULT_PROCFS = "/proc";
+ private DockerClient dockerClient;
+
@InterfaceAudience.Private
- public static final String ENV_DOCKER_CONTAINER_IMAGE =
- "YARN_CONTAINER_RUNTIME_DOCKER_IMAGE";
- @InterfaceAudience.Private
- public static final String ENV_DOCKER_CONTAINER_RUN_OVERRIDE_DISABLE =
- "YARN_CONTAINER_RUNTIME_DOCKER_RUN_OVERRIDE_DISABLE";
- @InterfaceAudience.Private
- public static final String ENV_DOCKER_CONTAINER_NETWORK =
+ static final String ENV_DOCKER_CONTAINER_NETWORK =
"YARN_CONTAINER_RUNTIME_DOCKER_CONTAINER_NETWORK";
- @InterfaceAudience.Private
- public static final String ENV_DOCKER_CONTAINER_PID_NAMESPACE =
- "YARN_CONTAINER_RUNTIME_DOCKER_CONTAINER_PID_NAMESPACE";
- @InterfaceAudience.Private
- public static final String ENV_DOCKER_CONTAINER_HOSTNAME =
- "YARN_CONTAINER_RUNTIME_DOCKER_CONTAINER_HOSTNAME";
- @InterfaceAudience.Private
- public static final String ENV_DOCKER_CONTAINER_RUN_PRIVILEGED_CONTAINER =
- "YARN_CONTAINER_RUNTIME_DOCKER_RUN_PRIVILEGED_CONTAINER";
- @InterfaceAudience.Private
- public static final String ENV_DOCKER_CONTAINER_MOUNTS =
- "YARN_CONTAINER_RUNTIME_DOCKER_MOUNTS";
- @InterfaceAudience.Private
- public static final String ENV_DOCKER_CONTAINER_TMPFS_MOUNTS =
- "YARN_CONTAINER_RUNTIME_DOCKER_TMPFS_MOUNTS";
- @InterfaceAudience.Private
- public static final String ENV_DOCKER_CONTAINER_DELAYED_REMOVAL =
- "YARN_CONTAINER_RUNTIME_DOCKER_DELAYED_REMOVAL";
- @InterfaceAudience.Private
- public static final String ENV_DOCKER_CONTAINER_PORTS_MAPPING =
- "YARN_CONTAINER_RUNTIME_DOCKER_PORTS_MAPPING";
- @InterfaceAudience.Private
- public static final String ENV_DOCKER_CONTAINER_YARN_SYSFS =
- "YARN_CONTAINER_RUNTIME_YARN_SYSFS_ENABLE";
- @InterfaceAudience.Private
- public static final String ENV_DOCKER_CONTAINER_DOCKER_RUNTIME =
- "YARN_CONTAINER_RUNTIME_DOCKER_RUNTIME";
- public static final String YARN_SYSFS_PATH =
- "/hadoop/yarn/sysfs";
- private Configuration conf;
- private Context nmContext;
- private DockerClient dockerClient;
- private Map csiClients = new HashMap<>();
- private PrivilegedOperationExecutor privilegedOperationExecutor;
- private String defaultImageName;
- private Boolean defaultImageUpdate;
- private Set allowedNetworks = new HashSet<>();
- private Set allowedRuntimes = new HashSet<>();
- private String defaultNetwork;
- private String defaultRuntime;
- private CGroupsHandler cGroupsHandler;
- private AccessControlList privilegedContainersAcl;
- private boolean enableUserReMapping;
- private int userRemappingUidThreshold;
- private int userRemappingGidThreshold;
- private Set capabilities;
- private boolean delayedRemovalAllowed;
- private Set defaultROMounts = new HashSet<>();
- private Set defaultRWMounts = new HashSet<>();
- private Set defaultTmpfsMounts = new HashSet<>();
/**
* Return whether the given environment variables indicate that the operation
@@ -338,130 +163,35 @@ public DockerLinuxContainerRuntime(PrivilegedOperationExecutor
public DockerLinuxContainerRuntime(
PrivilegedOperationExecutor privilegedOperationExecutor,
CGroupsHandler cGroupsHandler) {
- this.privilegedOperationExecutor = privilegedOperationExecutor;
-
- if (cGroupsHandler == null) {
- LOG.info("cGroupsHandler is null - cgroups not in use.");
- } else {
- this.cGroupsHandler = cGroupsHandler;
- }
+ super(privilegedOperationExecutor, cGroupsHandler);
}
@Override
public void initialize(Configuration conf, Context nmContext)
throws ContainerExecutionException {
- this.nmContext = nmContext;
- this.conf = conf;
+ super.initialize(conf, nmContext);
dockerClient = new DockerClient();
- allowedNetworks.clear();
- allowedRuntimes.clear();
- defaultROMounts.clear();
- defaultRWMounts.clear();
- defaultTmpfsMounts.clear();
- defaultImageName = conf.getTrimmed(
- YarnConfiguration.NM_DOCKER_IMAGE_NAME, "");
- defaultImageUpdate = conf.getBoolean(
- YarnConfiguration.NM_DOCKER_IMAGE_UPDATE, false);
- allowedNetworks.addAll(Arrays.asList(
- conf.getTrimmedStrings(
- YarnConfiguration.NM_DOCKER_ALLOWED_CONTAINER_NETWORKS,
- YarnConfiguration.DEFAULT_NM_DOCKER_ALLOWED_CONTAINER_NETWORKS)));
- defaultNetwork = conf.getTrimmed(
- YarnConfiguration.NM_DOCKER_DEFAULT_CONTAINER_NETWORK,
- YarnConfiguration.DEFAULT_NM_DOCKER_DEFAULT_CONTAINER_NETWORK);
- allowedRuntimes.addAll(Arrays.asList(
- conf.getTrimmedStrings(
- YarnConfiguration.NM_DOCKER_ALLOWED_CONTAINER_RUNTIMES,
- YarnConfiguration.DEFAULT_NM_DOCKER_ALLOWED_CONTAINER_RUNTIMES)));
-
- if(!allowedNetworks.contains(defaultNetwork)) {
- String message = "Default network: " + defaultNetwork
- + " is not in the set of allowed networks: " + allowedNetworks;
-
- if (LOG.isWarnEnabled()) {
- LOG.warn(message + ". Please check configuration");
- }
-
- throw new ContainerExecutionException(message);
- }
// initialize csi adaptors if necessary
initiateCsiClients(conf);
-
- privilegedContainersAcl = new AccessControlList(conf.getTrimmed(
- YarnConfiguration.NM_DOCKER_PRIVILEGED_CONTAINERS_ACL,
- YarnConfiguration.DEFAULT_NM_DOCKER_PRIVILEGED_CONTAINERS_ACL));
-
- enableUserReMapping = conf.getBoolean(
- YarnConfiguration.NM_DOCKER_ENABLE_USER_REMAPPING,
- YarnConfiguration.DEFAULT_NM_DOCKER_ENABLE_USER_REMAPPING);
-
- userRemappingUidThreshold = conf.getInt(
- YarnConfiguration.NM_DOCKER_USER_REMAPPING_UID_THRESHOLD,
- YarnConfiguration.DEFAULT_NM_DOCKER_USER_REMAPPING_UID_THRESHOLD);
-
- userRemappingGidThreshold = conf.getInt(
- YarnConfiguration.NM_DOCKER_USER_REMAPPING_GID_THRESHOLD,
- YarnConfiguration.DEFAULT_NM_DOCKER_USER_REMAPPING_GID_THRESHOLD);
-
- capabilities = getDockerCapabilitiesFromConf();
-
- delayedRemovalAllowed = conf.getBoolean(
- YarnConfiguration.NM_DOCKER_ALLOW_DELAYED_REMOVAL,
- YarnConfiguration.DEFAULT_NM_DOCKER_ALLOW_DELAYED_REMOVAL);
-
- defaultROMounts.addAll(Arrays.asList(
- conf.getTrimmedStrings(
- YarnConfiguration.NM_DOCKER_DEFAULT_RO_MOUNTS)));
-
- defaultRWMounts.addAll(Arrays.asList(
- conf.getTrimmedStrings(
- YarnConfiguration.NM_DOCKER_DEFAULT_RW_MOUNTS)));
-
- defaultTmpfsMounts.addAll(Arrays.asList(
- conf.getTrimmedStrings(
- YarnConfiguration.NM_DOCKER_DEFAULT_TMPFS_MOUNTS)));
- }
-
- public Map getCsiClients() {
- return csiClients;
}
@Override
public boolean isRuntimeRequested(Map env) {
- return isDockerContainerRequested(conf, env);
- }
-
- private Set getDockerCapabilitiesFromConf() throws
- ContainerExecutionException {
- Set caps = new HashSet<>(Arrays.asList(
- conf.getTrimmedStrings(
- YarnConfiguration.NM_DOCKER_CONTAINER_CAPABILITIES,
- YarnConfiguration.DEFAULT_NM_DOCKER_CONTAINER_CAPABILITIES)));
- if(caps.contains("none") || caps.contains("NONE")) {
- if(caps.size() > 1) {
- String msg = "Mixing capabilities with the none keyword is" +
- " not supported";
- throw new ContainerExecutionException(msg);
- }
- caps = Collections.emptySet();
- }
-
- return caps;
- }
-
- public Set getCapabilities() {
- return capabilities;
+ return isDockerContainerRequested(super.getConf(), env);
}
private String runDockerVolumeCommand(DockerVolumeCommand dockerVolumeCommand,
Container container) throws ContainerExecutionException {
try {
String commandFile = dockerClient.writeCommandToTempFile(
- dockerVolumeCommand, container.getContainerId(), nmContext);
+ dockerVolumeCommand, container.getContainerId(),
+ super.getNmContext());
PrivilegedOperation privOp = new PrivilegedOperation(
PrivilegedOperation.OperationType.RUN_DOCKER_CMD);
privOp.appendArgs(commandFile);
+ PrivilegedOperationExecutor privilegedOperationExecutor =
+ super.getPrivilegedOperationExecutor();
String output = privilegedOperationExecutor
.executePrivilegedOperation(null, privOp, null,
null, true, false);
@@ -479,12 +209,6 @@ private String runDockerVolumeCommand(DockerVolumeCommand dockerVolumeCommand,
+ dockerVolumeCommand, e);
throw new ContainerExecutionException(e);
}
-
- }
-
- @Override
- public void prepareContainer(ContainerRuntimeContext ctx)
- throws ContainerExecutionException {
}
private void checkDockerVolumeCreated(
@@ -527,87 +251,6 @@ private void checkDockerVolumeCreated(
throw new ContainerExecutionException(message);
}
- private void validateContainerNetworkType(String network)
- throws ContainerExecutionException {
- if (allowedNetworks.contains(network)) {
- return;
- }
-
- String msg = "Disallowed network: '" + network
- + "' specified. Allowed networks: are " + allowedNetworks
- .toString();
- throw new ContainerExecutionException(msg);
- }
-
- private void validateContainerRuntimeType(String runtime)
- throws ContainerExecutionException {
- if (runtime == null || runtime.isEmpty()
- || allowedRuntimes.contains(runtime)) {
- return;
- }
-
- String msg = "Disallowed runtime: '" + runtime
- + "' specified. Allowed networks: are " + allowedRuntimes
- .toString();
- throw new ContainerExecutionException(msg);
- }
-
- /**
- * Return whether the YARN container is allowed to run using the host's PID
- * namespace for the Docker container. For this to be allowed, the submitting
- * user must request the feature and the feature must be enabled on the
- * cluster.
- *
- * @param container the target YARN container
- * @return whether host pid namespace is requested and allowed
- * @throws ContainerExecutionException if host pid namespace is requested
- * but is not allowed
- */
- private boolean allowHostPidNamespace(Container container)
- throws ContainerExecutionException {
- Map environment = container.getLaunchContext()
- .getEnvironment();
- String pidNamespace = environment.get(ENV_DOCKER_CONTAINER_PID_NAMESPACE);
-
- if (pidNamespace == null) {
- return false;
- }
-
- if (!pidNamespace.equalsIgnoreCase("host")) {
- LOG.warn("NOT requesting PID namespace. Value of " +
- ENV_DOCKER_CONTAINER_PID_NAMESPACE + "is invalid: " + pidNamespace);
- return false;
- }
-
- boolean hostPidNamespaceEnabled = conf.getBoolean(
- YarnConfiguration.NM_DOCKER_ALLOW_HOST_PID_NAMESPACE,
- YarnConfiguration.DEFAULT_NM_DOCKER_ALLOW_HOST_PID_NAMESPACE);
-
- if (!hostPidNamespaceEnabled) {
- String message = "Host pid namespace being requested but this is not "
- + "enabled on this cluster";
- LOG.warn(message);
- throw new ContainerExecutionException(message);
- }
-
- return true;
- }
-
- public static void validateHostname(String hostname) throws
- ContainerExecutionException {
- if (hostname != null && !hostname.isEmpty()) {
- if (!hostnamePattern.matcher(hostname).matches()) {
- throw new ContainerExecutionException("Hostname '" + hostname
- + "' doesn't match docker hostname pattern");
- }
- if (hostname.length() > HOST_NAME_LENGTH) {
- throw new ContainerExecutionException(
- "Hostname can not be greater than " + HOST_NAME_LENGTH
- + " characters: " + hostname);
- }
- }
- }
-
/** Set a DNS friendly hostname.
* Only add hostname if network is not host or if hostname is
* specified via YARN_CONTAINER_RUNTIME_DOCKER_CONTAINER_HOSTNAME
@@ -627,7 +270,7 @@ private void setHostname(DockerRunCommand runCommand,
if (name == null || name.isEmpty()) {
name = RegistryPathUtils.encodeYarnID(containerIdStr);
- String domain = conf.get(RegistryConstants.KEY_DNS_DOMAIN);
+ String domain = super.getConf().get(RegistryConstants.KEY_DNS_DOMAIN);
if (domain != null) {
name += ("." + domain);
}
@@ -647,8 +290,9 @@ private void setHostname(DockerRunCommand runCommand,
* @param runCommand the command to set with the CGROUP parent
*/
@VisibleForTesting
- protected void addCGroupParentIfRequired(String resourcesOptions,
+ void addCGroupParentIfRequired(String resourcesOptions,
String containerIdStr, DockerRunCommand runCommand) {
+ CGroupsHandler cGroupsHandler = super.getcGroupsHandler();
if (cGroupsHandler == null) {
LOG.debug("cGroupsHandler is null. cgroups are not in use. nothing to"
+ " do.");
@@ -671,148 +315,6 @@ protected void addCGroupParentIfRequired(String resourcesOptions,
}
}
- /**
- * Return whether the YARN container is allowed to run in a privileged
- * Docker container. For a privileged container to be allowed all of the
- * following three conditions must be satisfied:
- *
- *
- * - Submitting user must request for a privileged container
- * - Privileged containers must be enabled on the cluster
- * - Submitting user must be white-listed to run a privileged
- * container
- *
- *
- * @param container the target YARN container
- * @return whether privileged container execution is allowed
- * @throws ContainerExecutionException if privileged container execution
- * is requested but is not allowed
- */
- private boolean allowPrivilegedContainerExecution(Container container)
- throws ContainerExecutionException {
-
- if(!isContainerRequestedAsPrivileged(container)) {
- return false;
- }
-
- LOG.info("Privileged container requested for : " + container
- .getContainerId().toString());
-
- //Ok, so we have been asked to run a privileged container. Security
- // checks need to be run. Each violation is an error.
-
- //check if privileged containers are enabled.
- boolean privilegedContainersEnabledOnCluster = conf.getBoolean(
- YarnConfiguration.NM_DOCKER_ALLOW_PRIVILEGED_CONTAINERS,
- YarnConfiguration.DEFAULT_NM_DOCKER_ALLOW_PRIVILEGED_CONTAINERS);
-
- if (!privilegedContainersEnabledOnCluster) {
- String message = "Privileged container being requested but privileged "
- + "containers are not enabled on this cluster";
- LOG.warn(message);
- throw new ContainerExecutionException(message);
- }
-
- //check if submitting user is in the whitelist.
- String submittingUser = container.getUser();
- UserGroupInformation submitterUgi = UserGroupInformation
- .createRemoteUser(submittingUser);
-
- if (!privilegedContainersAcl.isUserAllowed(submitterUgi)) {
- String message = "Cannot launch privileged container. Submitting user ("
- + submittingUser + ") fails ACL check.";
- LOG.warn(message);
- throw new ContainerExecutionException(message);
- }
-
- LOG.info("All checks pass. Launching privileged container for : "
- + container.getContainerId().toString());
-
- return true;
- }
-
- /**
- * This function only returns whether a privileged container was requested,
- * not whether the container was or will be launched as privileged.
- * @param container
- * @return
- */
- private boolean isContainerRequestedAsPrivileged(
- Container container) {
- String runPrivilegedContainerEnvVar = container.getLaunchContext()
- .getEnvironment().get(ENV_DOCKER_CONTAINER_RUN_PRIVILEGED_CONTAINER);
- return Boolean.parseBoolean(runPrivilegedContainerEnvVar);
- }
-
- @VisibleForTesting
- private String mountReadOnlyPath(String mount,
- Map> localizedResources)
- throws ContainerExecutionException {
- for (Entry> resource : localizedResources.entrySet()) {
- if (resource.getValue().contains(mount)) {
- java.nio.file.Path path = Paths.get(resource.getKey().toString());
- if (!path.isAbsolute()) {
- throw new ContainerExecutionException("Mount must be absolute: " +
- mount);
- }
- if (Files.isSymbolicLink(path)) {
- throw new ContainerExecutionException("Mount cannot be a symlink: " +
- mount);
- }
- return path.toString();
- }
- }
- throw new ContainerExecutionException("Mount must be a localized " +
- "resource: " + mount);
- }
-
- private String getUserIdInfo(String userName)
- throws ContainerExecutionException {
- String id = "";
- Shell.ShellCommandExecutor shexec = new Shell.ShellCommandExecutor(
- new String[]{"id", "-u", userName});
- try {
- shexec.execute();
- id = shexec.getOutput().replaceAll("[^0-9]", "");
- } catch (Exception e) {
- throw new ContainerExecutionException(e);
- }
- return id;
- }
-
- private String[] getGroupIdInfo(String userName)
- throws ContainerExecutionException {
- String[] id = null;
- Shell.ShellCommandExecutor shexec = new Shell.ShellCommandExecutor(
- new String[]{"id", "-G", userName});
- try {
- shexec.execute();
- id = shexec.getOutput().replace("\n", "").split(" ");
- } catch (Exception e) {
- throw new ContainerExecutionException(e);
- }
- return id;
- }
-
- /**
- * Check if system is default to disable docker override or
- * user requested a Docker container with ENTRY_POINT support.
- *
- * @param environment - Docker container environment variables
- * @return true if Docker launch command override is disabled
- */
- private boolean checkUseEntryPoint(Map environment) {
- boolean overrideDisable = false;
- String overrideDisableKey = Environment.
- YARN_CONTAINER_RUNTIME_DOCKER_RUN_OVERRIDE_DISABLE.
- name();
- String overrideDisableValue = (environment.get(overrideDisableKey) != null)
- ? environment.get(overrideDisableKey) :
- System.getenv(overrideDisableKey);
- overrideDisable = Boolean.parseBoolean(overrideDisableValue);
- return overrideDisable;
- }
-
@Override
public void launchContainer(ContainerRuntimeContext ctx)
throws ContainerExecutionException {
@@ -828,10 +330,10 @@ public void launchContainer(ContainerRuntimeContext ctx)
boolean useEntryPoint = checkUseEntryPoint(environment);
if (imageName == null || imageName.isEmpty()) {
- imageName = defaultImageName;
+ imageName = super.getDefaultImageName();
}
if(network == null || network.isEmpty()) {
- network = defaultNetwork;
+ network = super.getDefaultNetwork();
}
validateContainerNetworkType(network);
@@ -842,7 +344,7 @@ public void launchContainer(ContainerRuntimeContext ctx)
validateContainerRuntimeType(runtime);
- if (defaultImageUpdate) {
+ if (super.getDefaultImageUpdate()) {
pullImageFromRemote(containerIdStr, imageName);
}
@@ -851,7 +353,9 @@ public void launchContainer(ContainerRuntimeContext ctx)
Path containerWorkDir = ctx.getExecutionAttribute(CONTAINER_WORK_DIR);
String[] groups = null;
- if (enableUserReMapping) {
+ if (super.getEnableUserReMapping()) {
+ int userRemappingUidThreshold = super.getUserRemappingUidThreshold();
+ int userRemappingGidThreshold = super.getUserRemappingGidThreshold();
String uid = getUserIdInfo(runAsUser);
groups = getGroupIdInfo(runAsUser);
String gid = groups[0];
@@ -911,7 +415,7 @@ public void launchContainer(ContainerRuntimeContext ctx)
}
}
- runCommand.setCapabilities(capabilities);
+ runCommand.setCapabilities(super.getCapabilities());
if (runtime != null && !runtime.isEmpty()) {
runCommand.addRuntime(runtime);
}
@@ -957,6 +461,7 @@ public void launchContainer(ContainerRuntimeContext ctx)
}
}
+ Set defaultROMounts = super.getDefaultROMounts();
if(defaultROMounts != null && !defaultROMounts.isEmpty()) {
for (String mount : defaultROMounts) {
String[] dir = StringUtils.split(mount, ':');
@@ -970,6 +475,7 @@ public void launchContainer(ContainerRuntimeContext ctx)
}
}
+ Set defaultRWMounts = super.getDefaultRWMounts();
if(defaultRWMounts != null && !defaultRWMounts.isEmpty()) {
for (String mount : defaultRWMounts) {
String[] dir = StringUtils.split(mount, ':');
@@ -1007,6 +513,7 @@ public void launchContainer(ContainerRuntimeContext ctx)
}
}
+ Set defaultTmpfsMounts = getDefaultTmpfsMounts();
if (defaultTmpfsMounts != null && !defaultTmpfsMounts.isEmpty()) {
for (String mount : defaultTmpfsMounts) {
if (!TMPFS_MOUNT_PATTERN.matcher(mount).matches()) {
@@ -1055,12 +562,14 @@ public void launchContainer(ContainerRuntimeContext ctx)
runCommand.detachOnRun();
}
- if(enableUserReMapping) {
+ if(super.getEnableUserReMapping()) {
if (!allowPrivilegedContainerExecution(container)) {
runCommand.groupAdd(groups);
}
}
+ Context nmContext = super.getNmContext();
+
// use plugins to create volume and update docker run command.
if (nmContext != null
&& nmContext.getResourcePluginManager().getNameToPlugins() != null) {
@@ -1098,6 +607,9 @@ public void launchContainer(ContainerRuntimeContext ctx)
// Some failures here are acceptable. Let the calling executor decide.
launchOp.disableFailureLogging();
+ PrivilegedOperationExecutor privilegedOperationExecutor =
+ super.getPrivilegedOperationExecutor();
+
try {
privilegedOperationExecutor.executePrivilegedOperation(null,
launchOp, null, null, false, false);
@@ -1112,6 +624,10 @@ public void relaunchContainer(ContainerRuntimeContext ctx)
throws ContainerExecutionException {
ContainerId containerId = ctx.getContainer().getContainerId();
String containerIdStr = containerId.toString();
+ Context nmContext = super.getNmContext();
+ PrivilegedOperationExecutor privilegedOperationExecutor =
+ super.getPrivilegedOperationExecutor();
+
// Check to see if the container already exists for relaunch
DockerCommandExecutor.DockerContainerStatus containerStatus =
DockerCommandExecutor.getContainerStatus(containerIdStr,
@@ -1140,12 +656,11 @@ public void relaunchContainer(ContainerRuntimeContext ctx)
}
}
-
/**
* Signal the docker container.
*
* Signals are used to check the liveliness of the container as well as to
- * stop/kill the container. The following outlines the docker container
+ * stop/kill the container. The following outlines the Docker container
* signal handling.
*
*
@@ -1197,6 +712,7 @@ public void reapContainer(ContainerRuntimeContext ctx)
handleContainerRemove(ctx.getContainer().getContainerId().toString(),
ctx.getContainer().getLaunchContext().getEnvironment());
+ Context nmContext = super.getNmContext();
// Cleanup volumes when needed.
if (nmContext != null
&& nmContext.getResourcePluginManager().getNameToPlugins() != null) {
@@ -1217,7 +733,7 @@ public void reapContainer(ContainerRuntimeContext ctx)
}
/**
- * Perform docker exec command into running container
+ * Perform docker exec command into running container.
*
* @param ctx container exec context
* @return IOStreams of docker exec
@@ -1238,13 +754,15 @@ public IOStreamPair execContainer(ContainerExecContext ctx)
command.add("-i");
dockerExecCommand.setOverrideCommandWithArgs(command);
String commandFile = dockerClient.writeCommandToTempFile(dockerExecCommand,
- ContainerId.fromString(containerId), nmContext);
+ ContainerId.fromString(containerId), super.getNmContext());
PrivilegedOperation privOp = new PrivilegedOperation(
PrivilegedOperation.OperationType.EXEC_CONTAINER);
privOp.appendArgs(commandFile);
privOp.disableFailureLogging();
IOStreamPair output;
+ PrivilegedOperationExecutor privilegedOperationExecutor =
+ super.getPrivilegedOperationExecutor();
try {
output =
privilegedOperationExecutor.executePrivilegedInteractiveOperation(
@@ -1285,6 +803,7 @@ public IOStreamPair execContainer(ContainerExecContext ctx)
String host = output.substring(index+1).trim();
if (ips.equals("")) {
String network;
+ String defaultNetwork = super.getDefaultNetwork();
try {
network = container.getLaunchContext().getEnvironment()
.get("YARN_CONTAINER_RUNTIME_DOCKER_CONTAINER_NETWORK");
@@ -1322,8 +841,7 @@ public IOStreamPair execContainer(ContainerExecContext ctx)
}
@Override
- public String getExposedPorts(Container container)
- throws ContainerExecutionException {
+ public String getExposedPorts(Container container) {
ContainerId containerId = container.getContainerId();
String containerIdStr = containerId.toString();
DockerInspectCommand inspectCommand =
@@ -1391,28 +909,34 @@ private PrivilegedOperation buildLaunchOp(ContainerRuntimeContext ctx,
return launchOp;
}
- public static void validateImageName(String imageName)
+ private void executeLivelinessCheck(ContainerRuntimeContext ctx)
throws ContainerExecutionException {
- if (imageName == null || imageName.isEmpty()) {
- throw new ContainerExecutionException(
- ENV_DOCKER_CONTAINER_IMAGE + " not set!");
+ String procFs = ctx.getExecutionAttribute(PROCFS);
+ if (procFs == null || procFs.isEmpty()) {
+ procFs = DEFAULT_PROCFS;
}
- if (!dockerImagePattern.matcher(imageName).matches()) {
- throw new ContainerExecutionException("Image name '" + imageName
- + "' doesn't match docker image name pattern");
+ String pid = ctx.getExecutionAttribute(PID);
+ if (!new File(procFs + File.separator + pid).exists()) {
+ String msg = "Liveliness check failed for PID: " + pid
+ + ". Container may have already completed.";
+ throw new ContainerExecutionException(msg,
+ PrivilegedOperation.ResultCode.INVALID_CONTAINER_PID.getValue());
}
}
- public void pullImageFromRemote(String containerIdStr, String imageName)
+ private void pullImageFromRemote(String containerIdStr, String imageName)
throws ContainerExecutionException {
long start = System.currentTimeMillis();
DockerPullCommand dockerPullCommand = new DockerPullCommand(imageName);
LOG.debug("now pulling docker image. image name: {}, container: {}",
imageName, containerIdStr);
+ PrivilegedOperationExecutor privilegedOperationExecutor =
+ super.getPrivilegedOperationExecutor();
+
DockerCommandExecutor.executeDockerCommand(dockerPullCommand,
containerIdStr, null,
- privilegedOperationExecutor, false, nmContext);
+ privilegedOperationExecutor, false, super.getNmContext());
long end = System.currentTimeMillis();
long pullImageTimeMs = end - start;
@@ -1421,21 +945,6 @@ public void pullImageFromRemote(String containerIdStr, String imageName)
+ " container: {}", pullImageTimeMs, imageName, containerIdStr);
}
- private void executeLivelinessCheck(ContainerRuntimeContext ctx)
- throws ContainerExecutionException {
- String procFs = ctx.getExecutionAttribute(PROCFS);
- if (procFs == null || procFs.isEmpty()) {
- procFs = DEFAULT_PROCFS;
- }
- String pid = ctx.getExecutionAttribute(PID);
- if (!new File(procFs + File.separator + pid).exists()) {
- String msg = "Liveliness check failed for PID: " + pid
- + ". Container may have already completed.";
- throw new ContainerExecutionException(msg,
- PrivilegedOperation.ResultCode.INVALID_CONTAINER_PID.getValue());
- }
- }
-
/**
* Handles a docker container stop by first finding the {@code STOPSIGNAL}
* using docker inspect and then executing
@@ -1481,12 +990,14 @@ private void handleContainerStop(ContainerId containerId,
}
if (DockerCommandExecutor.isStoppable(containerStatus)) {
+ PrivilegedOperationExecutor privilegedOperationExecutor =
+ super.getPrivilegedOperationExecutor();
DockerKillCommand dockerStopCommand = new DockerKillCommand(
containerId.toString()).setSignal(stopSignal);
DockerCommandExecutor.executeDockerCommand(dockerStopCommand,
containerId.toString(), env, privilegedOperationExecutor, false,
- nmContext);
+ super.getNmContext());
} else {
LOG.debug("{} status is {}, skipping stop", containerId, containerStatus);
}
@@ -1496,10 +1007,14 @@ private String executeDockerInspect(ContainerId containerId,
DockerInspectCommand inspectCommand) throws ContainerExecutionException,
PrivilegedOperationException {
String commandFile = dockerClient.writeCommandToTempFile(inspectCommand,
- containerId, nmContext);
+ containerId, super.getNmContext());
PrivilegedOperation privOp = new PrivilegedOperation(
PrivilegedOperation.OperationType.RUN_DOCKER_CMD);
privOp.appendArgs(commandFile);
+
+ PrivilegedOperationExecutor privilegedOperationExecutor =
+ super.getPrivilegedOperationExecutor();
+
String output = privilegedOperationExecutor.executePrivilegedOperation(null,
privOp, null, null, true, false);
LOG.info("{} : docker inspect output {} ", containerId, output);
@@ -1519,11 +1034,15 @@ private void handleContainerKill(ContainerRuntimeContext ctx,
throw new ContainerExecutionException(e);
}
+ Context nmContext = super.getNmContext();
+
// Only need to check whether the container was asked to be privileged.
// If the container had failed the permissions checks upon launch, it
// would have never been launched and thus we wouldn't be here
// attempting to signal it.
if (isContainerRequestedAsPrivileged(container)) {
+ PrivilegedOperationExecutor privilegedOperationExecutor =
+ super.getPrivilegedOperationExecutor();
String containerId = container.getContainerId().toString();
DockerCommandExecutor.DockerContainerStatus containerStatus =
DockerCommandExecutor.getContainerStatus(containerId,
@@ -1549,6 +1068,8 @@ private void handleContainerKill(ContainerRuntimeContext ctx,
Integer.toString(ctx.getExecutionAttribute(SIGNAL).getValue()));
privOp.disableFailureLogging();
try {
+ PrivilegedOperationExecutor privilegedOperationExecutor =
+ super.getPrivilegedOperationExecutor();
privilegedOperationExecutor.executePrivilegedOperation(null,
privOp, null, null, false, false);
} catch (PrivilegedOperationException e) {
@@ -1561,14 +1082,17 @@ private void handleContainerKill(ContainerRuntimeContext ctx,
}
}
- private void handleContainerRemove(String containerId,
+ protected void handleContainerRemove(String containerId,
Map env) throws ContainerExecutionException {
String delayedRemoval = env.get(ENV_DOCKER_CONTAINER_DELAYED_REMOVAL);
- if (delayedRemovalAllowed && delayedRemoval != null
+ if (super.isDelayedRemovalAllowed() && delayedRemoval != null
&& delayedRemoval.equalsIgnoreCase("true")) {
LOG.info("Delayed removal requested and allowed, skipping removal - "
+ containerId);
} else {
+ PrivilegedOperationExecutor privilegedOperationExecutor =
+ super.getPrivilegedOperationExecutor();
+ Context nmContext = super.getNmContext();
DockerCommandExecutor.DockerContainerStatus containerStatus =
DockerCommandExecutor.getContainerStatus(containerId,
privilegedOperationExecutor, nmContext);
@@ -1613,34 +1137,4 @@ private void addDockerClientConfigToRunCommand(ContainerRuntimeContext ctx,
}
}
}
-
- /**
- * Initiate CSI clients to talk to the CSI adaptors on this node and
- * cache the clients for easier fetch.
- * @param config configuration
- * @throws ContainerExecutionException
- */
- private void initiateCsiClients(Configuration config)
- throws ContainerExecutionException {
- String[] driverNames = CsiConfigUtils.getCsiDriverNames(config);
- if (driverNames != null && driverNames.length > 0) {
- for (String driverName : driverNames) {
- try {
- // find out the adaptors service address
- InetSocketAddress adaptorServiceAddress =
- CsiConfigUtils.getCsiAdaptorAddressForDriver(driverName, config);
- LOG.info("Initializing a csi-adaptor-client for csi-adaptor {},"
- + " csi-driver {}", adaptorServiceAddress.toString(), driverName);
- CsiAdaptorProtocolPBClientImpl client =
- new CsiAdaptorProtocolPBClientImpl(1L, adaptorServiceAddress,
- config);
- csiClients.put(driverName, client);
- } catch (IOException e1) {
- throw new ContainerExecutionException(e1.getMessage());
- } catch (YarnException e2) {
- throw new ContainerExecutionException(e2.getMessage());
- }
- }
- }
- }
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/OCIContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/OCIContainerRuntime.java
new file mode 100644
index 00000000000..0f3b1b7ba72
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/OCIContainerRuntime.java
@@ -0,0 +1,683 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.authorize.AccessControlList;
+import org.apache.hadoop.util.Shell;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.api.CsiAdaptorProtocol;
+import org.apache.hadoop.yarn.api.impl.pb.client.CsiAdaptorProtocolPBClientImpl;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.nodemanager.Context;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationExecutor;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.CGroupsHandler;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerModule;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerExecutionException;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerRuntime;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerRuntimeConstants;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerRuntimeContext;
+import org.apache.hadoop.yarn.util.csi.CsiConfigUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.regex.Pattern;
+
+/**
+ * This class is a {@link ContainerRuntime} implementation that uses the
+ * native {@code container-executor} binary via a
+ * {@link PrivilegedOperationExecutor} instance to launch processes inside
+ * OCI-compliant containers.
+ *
+ * The following environment variables are used to configure the OCI
+ * Container engine:
+ *
+ *
+ * -
+ * {@code YARN_CONTAINER_RUNTIME_DOCKER_IMAGE} names which image
+ * will be used to launch the OCI-compliant container.
+ *
+ * -
+ * {@code YARN_CONTAINER_RUNTIME_DOCKER_PORTS_MAPPING} allows users to
+ * specify ports mapping for the bridge network OCI-compliant container.
+ * The value of the environment variable should be a comma-separated list
+ * of ports mapping.
+ *
+ * For Docker containers: It's the same to "-p" option for the
+ * Docker run command. If the value is empty, "-P" will be added.
+ *
+ * -
+ * {@code YARN_CONTAINER_RUNTIME_DOCKER_CONTAINER_PID_NAMESPACE}
+ * controls which PID namespace will be used by the OCI-compliant container.
+ * By default, each OCI-compliant container has its own PID namespace.
+ * To share the namespace of the host, the
+ * {@code yarn.nodemanager.runtime.linux.docker.host-pid-namespace.allowed}
+ * property must be set to {@code true}. If the host PID namespace is
+ * allowed and this environment variable is set to {@code host}, the
+ * OCI-compliant container will share the host's PID namespace.
+ * No other value is allowed.
+ *
+ * -
+ * {@code YARN_CONTAINER_RUNTIME_DOCKER_CONTAINER_HOSTNAME} sets the
+ * hostname to be used by the OCI-compliant container. If not specified, a
+ * hostname will be derived from the container ID and set as default
+ * hostname for networks other than 'host'.
+ *
+ * -
+ * {@code YARN_CONTAINER_RUNTIME_DOCKER_RUN_PRIVILEGED_CONTAINER}
+ * controls whether the OCI-compliant container is a privileged container.
+ * In order to use privileged containers, the
+ * {@code yarn.nodemanager.runtime.linux.docker.privileged-containers.allowed}
+ * property must be set to {@code true}, and the application owner must
+ * appear in the value of the
+ * {@code yarn.nodemanager.runtime.linux.docker.privileged-containers.acl}
+ * property. If this environment variable is set to {@code true}, a
+ * privileged OCI-compliant container will be used if allowed. No other
+ * value is allowed, so the environment variable should be left unset
+ * rather than setting it to false.
+ *
+ * -
+ * {@code YARN_CONTAINER_RUNTIME_DOCKER_MOUNTS} allows users to specify
+ * additional volume mounts for the OCI-compliant container. The value of
+ * the environment variable should be a comma-separated list of mounts.
+ * All such mounts must be given as {@code source:dest[:mode]} and the mode
+ * must be "ro" (read-only) or "rw" (read-write) to specify the type of
+ * access being requested. If neither is specified, read-write will be
+ * assumed. The mode may include a bind propagation option. In that case,
+ * the mode should either be of the form [option], rw+[option], or
+ * ro+[option]. Valid bind propagation options are shared, rshared, slave,
+ * rslave, private, and rprivate. The requested mounts will be validated by
+ * container-executor based on the values set in container-executor.cfg for
+ * {@code docker.allowed.ro-mounts} and {@code docker.allowed.rw-mounts}.
+ *
+ * -
+ * {@code YARN_CONTAINER_RUNTIME_DOCKER_TMPFS_MOUNTS} allows users to
+ * specify additional tmpfs mounts for the OCI-compliant container.
+ * The value of the environment variable should be a comma-separated
+ * list of mounts.
+ *
+ * -
+ * {@code YARN_CONTAINER_RUNTIME_DOCKER_DELAYED_REMOVAL} allows a user
+ * to request delayed deletion of the OCI-compliant containers on a per
+ * container basis. If true, OCI-compliant containers will not be removed
+ * until the duration defined by
+ * {@code yarn.nodemanager.delete.debug-delay-sec}
+ * has elapsed. Administrators can disable this feature through the
+ * yarn-site property
+ * {@code yarn.nodemanager.runtime.linux.docker.delayed-removal.allowed}.
+ * This feature is disabled by default. When this feature is disabled or set
+ * to false, the container will be removed as soon as it exits.
+ *
+ * -
+ * {@code YARN_CONTAINER_RUNTIME_YARN_SYSFS_ENABLE} allows export yarn
+ * service json to OCI-compliant container. This feature is disabled by
+ * default. When this feature is set, app.json will be available in
+ * /hadoop/yarn/sysfs/app.json.
+ *
+ *
+ */
+
+public abstract class OCIContainerRuntime implements LinuxContainerRuntime {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(OCIContainerRuntime.class);
+
+ // This validates that the image is a proper docker image
+ private static final String DOCKER_IMAGE_PATTERN =
+ "^(([a-zA-Z0-9.-]+)(:\\d+)?/)?([a-z0-9_./-]+)(:[\\w.-]+)?$";
+ private static final Pattern dockerImagePattern =
+ Pattern.compile(DOCKER_IMAGE_PATTERN);
+ private static final String HOSTNAME_PATTERN =
+ "^[a-zA-Z0-9][a-zA-Z0-9_.-]+$";
+ private static final Pattern hostnamePattern = Pattern.compile(
+ HOSTNAME_PATTERN);
+ static final Pattern USER_MOUNT_PATTERN = Pattern.compile(
+ "(?<=^|,)([^:\\x00]+):([^:\\x00]+)" +
+ "(:(r[ow]|(r[ow][+])?(r?shared|r?slave|r?private)))?(?:,|$)");
+ static final Pattern TMPFS_MOUNT_PATTERN = Pattern.compile(
+ "^/[^:\\x00]+$");
+ static final String PORTS_MAPPING_PATTERN =
+ "^:[0-9]+|^[0-9]+:[0-9]+|^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]" +
+ "|25[0-5])\\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])" +
+ ":[0-9]+:[0-9]+$";
+ private static final int HOST_NAME_LENGTH = 64;
+
+ @InterfaceAudience.Private
+ static final String ENV_DOCKER_CONTAINER_IMAGE =
+ "YARN_CONTAINER_RUNTIME_DOCKER_IMAGE";
+
+ @InterfaceAudience.Private
+ static final String ENV_DOCKER_CONTAINER_PID_NAMESPACE =
+ "YARN_CONTAINER_RUNTIME_DOCKER_CONTAINER_PID_NAMESPACE";
+ @InterfaceAudience.Private
+ static final String ENV_DOCKER_CONTAINER_HOSTNAME =
+ "YARN_CONTAINER_RUNTIME_DOCKER_CONTAINER_HOSTNAME";
+ @InterfaceAudience.Private
+ static final String ENV_DOCKER_CONTAINER_RUN_PRIVILEGED_CONTAINER =
+ "YARN_CONTAINER_RUNTIME_DOCKER_RUN_PRIVILEGED_CONTAINER";
+ @InterfaceAudience.Private
+ static final String ENV_DOCKER_CONTAINER_MOUNTS =
+ "YARN_CONTAINER_RUNTIME_DOCKER_MOUNTS";
+ @InterfaceAudience.Private
+ static final String ENV_DOCKER_CONTAINER_TMPFS_MOUNTS =
+ "YARN_CONTAINER_RUNTIME_DOCKER_TMPFS_MOUNTS";
+ @InterfaceAudience.Private
+ static final String ENV_DOCKER_CONTAINER_DELAYED_REMOVAL =
+ "YARN_CONTAINER_RUNTIME_DOCKER_DELAYED_REMOVAL";
+ @InterfaceAudience.Private
+ static final String ENV_DOCKER_CONTAINER_PORTS_MAPPING =
+ "YARN_CONTAINER_RUNTIME_DOCKER_PORTS_MAPPING";
+ @InterfaceAudience.Private
+ static final String ENV_DOCKER_CONTAINER_YARN_SYSFS =
+ "YARN_CONTAINER_RUNTIME_YARN_SYSFS_ENABLE";
+ @InterfaceAudience.Private
+ static final String ENV_DOCKER_CONTAINER_DOCKER_RUNTIME =
+ "YARN_CONTAINER_RUNTIME_DOCKER_RUNTIME";
+
+ private Configuration conf;
+ private Context nmContext;
+ private Map csiClients = new HashMap<>();
+ private PrivilegedOperationExecutor privilegedOperationExecutor;
+ private String defaultImageName;
+ private Boolean defaultImageUpdate;
+ private Set allowedNetworks = new HashSet<>();
+ private Set allowedRuntimes = new HashSet<>();
+ private String defaultNetwork;
+ private CGroupsHandler cGroupsHandler;
+ private AccessControlList privilegedContainersAcl;
+ private boolean enableUserReMapping;
+ private int userRemappingUidThreshold;
+ private int userRemappingGidThreshold;
+ private Set capabilities;
+ private boolean delayedRemovalAllowed;
+ private Set defaultROMounts = new HashSet<>();
+ private Set defaultRWMounts = new HashSet<>();
+ private Set defaultTmpfsMounts = new HashSet<>();
+
+ public Configuration getConf() {
+ return conf;
+ }
+
+ public Context getNmContext() {
+ return nmContext;
+ }
+
+ public PrivilegedOperationExecutor getPrivilegedOperationExecutor() {
+ return privilegedOperationExecutor;
+ }
+
+ public String getDefaultImageName() {
+ return defaultImageName;
+ }
+
+ public Boolean getDefaultImageUpdate() {
+ return defaultImageUpdate;
+ }
+
+ public String getDefaultNetwork() {
+ return defaultNetwork;
+ }
+
+ public CGroupsHandler getcGroupsHandler() {
+ return cGroupsHandler;
+ }
+
+ public boolean getEnableUserReMapping() {
+ return enableUserReMapping;
+ }
+
+ public int getUserRemappingUidThreshold() {
+ return userRemappingUidThreshold;
+ }
+
+ public int getUserRemappingGidThreshold() {
+ return userRemappingGidThreshold;
+ }
+
+ public boolean isDelayedRemovalAllowed() {
+ return delayedRemovalAllowed;
+ }
+
+ public Set getDefaultROMounts() {
+ return defaultROMounts;
+ }
+
+ public Set getDefaultRWMounts() {
+ return defaultRWMounts;
+ }
+
+ public Set getDefaultTmpfsMounts() {
+ return defaultTmpfsMounts;
+ }
+
+ public OCIContainerRuntime(PrivilegedOperationExecutor
+ privilegedOperationExecutor) {
+ this(privilegedOperationExecutor, ResourceHandlerModule
+ .getCGroupsHandler());
+ }
+
+ public OCIContainerRuntime(PrivilegedOperationExecutor
+ privilegedOperationExecutor, CGroupsHandler cGroupsHandler) {
+ this.privilegedOperationExecutor = privilegedOperationExecutor;
+
+ if (cGroupsHandler == null) {
+ LOG.info("cGroupsHandler is null - cgroups not in use.");
+ } else {
+ this.cGroupsHandler = cGroupsHandler;
+ }
+ }
+
+ public void initialize(Configuration conf, Context nmContext)
+ throws ContainerExecutionException {
+ this.nmContext = nmContext;
+ this.conf = conf;
+
+ allowedNetworks.clear();
+ allowedRuntimes.clear();
+ defaultROMounts.clear();
+ defaultRWMounts.clear();
+ defaultTmpfsMounts.clear();
+ defaultImageName = conf.getTrimmed(
+ YarnConfiguration.NM_DOCKER_IMAGE_NAME, "");
+ defaultImageUpdate = conf.getBoolean(
+ YarnConfiguration.NM_DOCKER_IMAGE_UPDATE, false);
+ allowedNetworks.addAll(Arrays.asList(
+ conf.getTrimmedStrings(
+ YarnConfiguration.NM_DOCKER_ALLOWED_CONTAINER_NETWORKS,
+ YarnConfiguration.DEFAULT_NM_DOCKER_ALLOWED_CONTAINER_NETWORKS)));
+ defaultNetwork = conf.getTrimmed(
+ YarnConfiguration.NM_DOCKER_DEFAULT_CONTAINER_NETWORK,
+ YarnConfiguration.DEFAULT_NM_DOCKER_DEFAULT_CONTAINER_NETWORK);
+ allowedRuntimes.addAll(Arrays.asList(
+ conf.getTrimmedStrings(
+ YarnConfiguration.NM_DOCKER_ALLOWED_CONTAINER_RUNTIMES,
+ YarnConfiguration.DEFAULT_NM_DOCKER_ALLOWED_CONTAINER_RUNTIMES)));
+
+ if(!allowedNetworks.contains(defaultNetwork)) {
+ String message = "Default network: " + defaultNetwork
+ + " is not in the set of allowed networks: " + allowedNetworks;
+
+ if (LOG.isWarnEnabled()) {
+ LOG.warn(message + ". Please check configuration");
+ }
+
+ throw new ContainerExecutionException(message);
+ }
+
+ privilegedContainersAcl = new AccessControlList(conf.getTrimmed(
+ YarnConfiguration.NM_DOCKER_PRIVILEGED_CONTAINERS_ACL,
+ YarnConfiguration.DEFAULT_NM_DOCKER_PRIVILEGED_CONTAINERS_ACL));
+
+ enableUserReMapping = conf.getBoolean(
+ YarnConfiguration.NM_DOCKER_ENABLE_USER_REMAPPING,
+ YarnConfiguration.DEFAULT_NM_DOCKER_ENABLE_USER_REMAPPING);
+
+ userRemappingUidThreshold = conf.getInt(
+ YarnConfiguration.NM_DOCKER_USER_REMAPPING_UID_THRESHOLD,
+ YarnConfiguration.DEFAULT_NM_DOCKER_USER_REMAPPING_UID_THRESHOLD);
+
+ userRemappingGidThreshold = conf.getInt(
+ YarnConfiguration.NM_DOCKER_USER_REMAPPING_GID_THRESHOLD,
+ YarnConfiguration.DEFAULT_NM_DOCKER_USER_REMAPPING_GID_THRESHOLD);
+
+ capabilities = getDockerCapabilitiesFromConf();
+
+ delayedRemovalAllowed = conf.getBoolean(
+ YarnConfiguration.NM_DOCKER_ALLOW_DELAYED_REMOVAL,
+ YarnConfiguration.DEFAULT_NM_DOCKER_ALLOW_DELAYED_REMOVAL);
+
+ defaultROMounts.addAll(Arrays.asList(
+ conf.getTrimmedStrings(
+ YarnConfiguration.NM_DOCKER_DEFAULT_RO_MOUNTS)));
+
+ defaultRWMounts.addAll(Arrays.asList(
+ conf.getTrimmedStrings(
+ YarnConfiguration.NM_DOCKER_DEFAULT_RW_MOUNTS)));
+
+ defaultTmpfsMounts.addAll(Arrays.asList(
+ conf.getTrimmedStrings(
+ YarnConfiguration.NM_DOCKER_DEFAULT_TMPFS_MOUNTS)));
+ }
+
+ public static boolean isOCICompliantContainerRequested(
+ Configuration daemonConf, Map env) {
+ String type = (env == null)
+ ? null : env.get(ContainerRuntimeConstants.ENV_CONTAINER_TYPE);
+ if (type == null) {
+ type = daemonConf.get(YarnConfiguration.LINUX_CONTAINER_RUNTIME_TYPE);
+ }
+ return type != null && type.equals(
+ ContainerRuntimeConstants.CONTAINER_RUNTIME_DOCKER);
+ }
+
+ private Set getDockerCapabilitiesFromConf() throws
+ ContainerExecutionException {
+ Set caps = new HashSet<>(Arrays.asList(
+ conf.getTrimmedStrings(
+ YarnConfiguration.NM_DOCKER_CONTAINER_CAPABILITIES,
+ YarnConfiguration.DEFAULT_NM_DOCKER_CONTAINER_CAPABILITIES)));
+ if(caps.contains("none") || caps.contains("NONE")) {
+ if(caps.size() > 1) {
+ String msg = "Mixing capabilities with the none keyword is" +
+ " not supported";
+ throw new ContainerExecutionException(msg);
+ }
+ caps = Collections.emptySet();
+ }
+
+ return caps;
+ }
+
+ protected Set getCapabilities() {
+ return capabilities;
+ }
+
+ @VisibleForTesting
+ protected String mountReadOnlyPath(String mount,
+ Map> localizedResources)
+ throws ContainerExecutionException {
+ for (Map.Entry> resource :
+ localizedResources.entrySet()) {
+ if (resource.getValue().contains(mount)) {
+ java.nio.file.Path path = Paths.get(resource.getKey().toString());
+ if (!path.isAbsolute()) {
+ throw new ContainerExecutionException("Mount must be absolute: " +
+ mount);
+ }
+ if (Files.isSymbolicLink(path)) {
+ throw new ContainerExecutionException("Mount cannot be a symlink: " +
+ mount);
+ }
+ return path.toString();
+ }
+ }
+ throw new ContainerExecutionException("Mount must be a localized " +
+ "resource: " + mount);
+ }
+
+ @Override
+ public void prepareContainer(ContainerRuntimeContext ctx)
+ throws ContainerExecutionException {
+ }
+
+ protected abstract void handleContainerRemove(String containerId,
+ Map env) throws ContainerExecutionException;
+
+ protected String getUserIdInfo(String userName)
+ throws ContainerExecutionException {
+ String id;
+ Shell.ShellCommandExecutor shexec = new Shell.ShellCommandExecutor(
+ new String[]{"id", "-u", userName});
+ try {
+ shexec.execute();
+ id = shexec.getOutput().replaceAll("[^0-9]", "");
+ } catch (Exception e) {
+ throw new ContainerExecutionException(e);
+ }
+ return id;
+ }
+
+ protected String[] getGroupIdInfo(String userName)
+ throws ContainerExecutionException {
+ String[] id;
+ Shell.ShellCommandExecutor shexec = new Shell.ShellCommandExecutor(
+ new String[]{"id", "-G", userName});
+ try {
+ shexec.execute();
+ id = shexec.getOutput().replace("\n", "").split(" ");
+ } catch (Exception e) {
+ throw new ContainerExecutionException(e);
+ }
+ return id;
+ }
+
+ protected void validateContainerNetworkType(String network)
+ throws ContainerExecutionException {
+ if (allowedNetworks.contains(network)) {
+ return;
+ }
+
+ String msg = "Disallowed network: '" + network
+ + "' specified. Allowed networks: are " + allowedNetworks
+ .toString();
+ throw new ContainerExecutionException(msg);
+ }
+
+ protected void validateContainerRuntimeType(String runtime)
+ throws ContainerExecutionException {
+ if (runtime == null || runtime.isEmpty()
+ || allowedRuntimes.contains(runtime)) {
+ return;
+ }
+
+ String msg = "Disallowed runtime: '" + runtime
+ + "' specified. Allowed networks: are " + allowedRuntimes
+ .toString();
+ throw new ContainerExecutionException(msg);
+ }
+
+ /**
+ * Return whether the YARN container is allowed to run using the host's PID
+ * namespace for the OCI-compliant container. For this to be allowed, the
+ * submitting user must request the feature and the feature must be enabled
+ * on the cluster.
+ *
+ * @param container the target YARN container
+ * @return whether host pid namespace is requested and allowed
+ * @throws ContainerExecutionException if host pid namespace is requested
+ * but is not allowed
+ */
+ protected boolean allowHostPidNamespace(Container container)
+ throws ContainerExecutionException {
+ Map environment = container.getLaunchContext()
+ .getEnvironment();
+ String pidNamespace = environment.get(ENV_DOCKER_CONTAINER_PID_NAMESPACE);
+
+ if (pidNamespace == null) {
+ return false;
+ }
+
+ if (!pidNamespace.equalsIgnoreCase("host")) {
+ LOG.warn("NOT requesting PID namespace. Value of " +
+ ENV_DOCKER_CONTAINER_PID_NAMESPACE + "is invalid: " + pidNamespace);
+ return false;
+ }
+
+ boolean hostPidNamespaceEnabled = conf.getBoolean(
+ YarnConfiguration.NM_DOCKER_ALLOW_HOST_PID_NAMESPACE,
+ YarnConfiguration.DEFAULT_NM_DOCKER_ALLOW_HOST_PID_NAMESPACE);
+
+ if (!hostPidNamespaceEnabled) {
+ String message = "Host pid namespace being requested but this is not "
+ + "enabled on this cluster";
+ LOG.warn(message);
+ throw new ContainerExecutionException(message);
+ }
+
+ return true;
+ }
+
+ protected static void validateHostname(String hostname) throws
+ ContainerExecutionException {
+ if (hostname != null && !hostname.isEmpty()) {
+ if (!hostnamePattern.matcher(hostname).matches()) {
+ throw new ContainerExecutionException("Hostname '" + hostname
+ + "' doesn't match OCI-compliant hostname pattern");
+ }
+ if (hostname.length() > HOST_NAME_LENGTH) {
+ throw new ContainerExecutionException(
+ "Hostname can not be greater than " + HOST_NAME_LENGTH
+ + " characters: " + hostname);
+ }
+ }
+ }
+
+ /**
+ * Return whether the YARN container is allowed to run in a privileged
+ * OCI-compliant container. For a privileged container to be allowed all of
+ * the following three conditions must be satisfied:
+ *
+ *
+ * - Submitting user must request for a privileged container
+ * - Privileged containers must be enabled on the cluster
+ * - Submitting user must be white-listed to run a privileged
+ * container
+ *
+ *
+ * @param container the target YARN container
+ * @return whether privileged container execution is allowed
+ * @throws ContainerExecutionException if privileged container execution
+ * is requested but is not allowed
+ */
+ protected boolean allowPrivilegedContainerExecution(Container container)
+ throws ContainerExecutionException {
+
+ if(!isContainerRequestedAsPrivileged(container)) {
+ return false;
+ }
+
+ LOG.info("Privileged container requested for : " + container
+ .getContainerId().toString());
+
+ //Ok, so we have been asked to run a privileged container. Security
+ // checks need to be run. Each violation is an error.
+
+ //check if privileged containers are enabled.
+ boolean privilegedContainersEnabledOnCluster = conf.getBoolean(
+ YarnConfiguration.NM_DOCKER_ALLOW_PRIVILEGED_CONTAINERS,
+ YarnConfiguration.DEFAULT_NM_DOCKER_ALLOW_PRIVILEGED_CONTAINERS);
+
+ if (!privilegedContainersEnabledOnCluster) {
+ String message = "Privileged container being requested but privileged "
+ + "containers are not enabled on this cluster";
+ LOG.warn(message);
+ throw new ContainerExecutionException(message);
+ }
+
+ //check if submitting user is in the whitelist.
+ String submittingUser = container.getUser();
+ UserGroupInformation submitterUgi = UserGroupInformation
+ .createRemoteUser(submittingUser);
+
+ if (!privilegedContainersAcl.isUserAllowed(submitterUgi)) {
+ String message = "Cannot launch privileged container. Submitting user ("
+ + submittingUser + ") fails ACL check.";
+ LOG.warn(message);
+ throw new ContainerExecutionException(message);
+ }
+
+ LOG.info("All checks pass. Launching privileged container for : "
+ + container.getContainerId().toString());
+
+ return true;
+ }
+
+ /**
+ * This function only returns whether a privileged container was requested,
+ * not whether the container was or will be launched as privileged.
+ * @param container
+ * @return true if container is requested as privileged
+ */
+ protected boolean isContainerRequestedAsPrivileged(
+ Container container) {
+ String runPrivilegedContainerEnvVar = container.getLaunchContext()
+ .getEnvironment().get(ENV_DOCKER_CONTAINER_RUN_PRIVILEGED_CONTAINER);
+ return Boolean.parseBoolean(runPrivilegedContainerEnvVar);
+ }
+
+ /**
+ * Check if system is default to disable container launch override or
+ * user requested an OCI-compliant container with ENTRY_POINT support.
+ *
+ * @param environment - OCI-compliant container environment variables
+ * @return true if OCI-compliant launch command override is disabled
+ */
+ protected boolean checkUseEntryPoint(Map environment) {
+ boolean overrideDisable = false;
+ String overrideDisableKey = ApplicationConstants.Environment.
+ YARN_CONTAINER_RUNTIME_DOCKER_RUN_OVERRIDE_DISABLE.
+ name();
+ String overrideDisableValue = (environment.get(overrideDisableKey) != null)
+ ? environment.get(overrideDisableKey) :
+ System.getenv(overrideDisableKey);
+ overrideDisable = Boolean.parseBoolean(overrideDisableValue);
+ return overrideDisable;
+ }
+
+ protected static void validateImageName(String imageName)
+ throws ContainerExecutionException {
+ if (imageName == null || imageName.isEmpty()) {
+ throw new ContainerExecutionException(
+ ENV_DOCKER_CONTAINER_IMAGE + " not set!");
+ }
+ if (!dockerImagePattern.matcher(imageName).matches()) {
+ throw new ContainerExecutionException("Image name '" + imageName
+ + "' doesn't match docker image name pattern");
+ }
+ }
+
+ public Map getCsiClients() {
+ return csiClients;
+ }
+
+ /**
+ * Initiate CSI clients to talk to the CSI adaptors on this node and
+ * cache the clients for easier fetch.
+ * @param config configuration
+ * @throws ContainerExecutionException
+ */
+ protected void initiateCsiClients(Configuration config)
+ throws ContainerExecutionException {
+ String[] driverNames = CsiConfigUtils.getCsiDriverNames(config);
+ if (driverNames != null && driverNames.length > 0) {
+ for (String driverName : driverNames) {
+ try {
+ // find out the adaptors service address
+ InetSocketAddress adaptorServiceAddress =
+ CsiConfigUtils.getCsiAdaptorAddressForDriver(driverName, config);
+ LOG.info("Initializing a csi-adaptor-client for csi-adaptor {},"
+ + " csi-driver {}", adaptorServiceAddress.toString(), driverName);
+ CsiAdaptorProtocolPBClientImpl client =
+ new CsiAdaptorProtocolPBClientImpl(1L, adaptorServiceAddress,
+ config);
+ csiClients.put(driverName, client);
+ } catch (IOException | YarnException e1) {
+ throw new ContainerExecutionException(e1.getMessage());
+ }
+ }
+ }
+ }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/deviceframework/DeviceResourceHandlerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/deviceframework/DeviceResourceHandlerImpl.java
index 97ff94f7880..03a22f2d743 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/deviceframework/DeviceResourceHandlerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/deviceframework/DeviceResourceHandlerImpl.java
@@ -36,7 +36,7 @@
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.CGroupsHandler;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandler;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerException;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.DockerLinuxContainerRuntime;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.OCIContainerRuntime;
import java.io.IOException;
import java.util.ArrayList;
@@ -153,7 +153,7 @@ public DeviceResourceHandlerImpl(String resName,
cGroupsHandler.createCGroup(CGroupsHandler.CGroupController.DEVICES,
containerIdStr);
// non-Docker, use cgroups to do isolation
- if (!DockerLinuxContainerRuntime.isDockerContainerRequested(
+ if (!OCIContainerRuntime.isOCICompliantContainerRequested(
nmContext.getConf(),
container.getLaunchContext().getEnvironment())) {
tryIsolateDevices(allocation, containerIdStr);