diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 46e3323..09c3564 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -1461,6 +1461,18 @@ public static boolean isAclEnabled(Configuration conf) { public static final String DEFAULT_NM_DOCKER_DEFAULT_CONTAINER_NETWORK = "host"; + /** Allow administrators to disable the ability for users to keep docker + * containers on exit. Keeping the docker container on exit is provided for + * debug purposes only. + */ + public static final String NM_DOCKER_ALLOW_KEEP_CONTAINER_ON_EXIT = + DOCKER_CONTAINER_RUNTIME_PREFIX + "allow-keep-container-on-exit"; + + /** Default value for allowing users to keep containers on exit. + */ + public static final boolean DEFAULT_NM_DOCKER_ALLOW_KEEP_CONTAINER_ON_EXIT = + false; + /** The path to the Linux container executor.*/ public static final String NM_LINUX_CONTAINER_EXECUTOR_PATH = NM_PREFIX + "linux-container-executor.path"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index e956507..7586ef8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -1558,6 +1558,13 @@ + Administrator toggle to determine if users are allowed to + keep exited docker containers for debugging purposes. + yarn.nodemanager.runtime.linux.docker.allow-keep-container-on-exit + false + + + This flag determines whether memory limit will be set for the Windows Job Object of the containers launched by the default container executor. yarn.nodemanager.windows-container.memory-limit.enabled diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java index a3aff2f..ded4b74 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java @@ -40,9 +40,11 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.CGroupsHandler; import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerModule; import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker.DockerClient; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker.DockerCommandExecutor; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker.DockerContainerStatusHandler; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker.DockerContainerStatusHandler.DockerContainerStatus; import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker.DockerInspectCommand; import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker.DockerRunCommand; -import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker.DockerStopCommand; import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerExecutionException; import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerRuntime; import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerRuntimeConstants; @@ -120,6 +122,12 @@ * source is an absolute path that is not a symlink and that points to a * localized resource. * + *
  • + * {@code YARN_CONTAINER_RUNTIME_DOCKER_KEEP_CONTAINER_ON_EXIT} controls + * whether a container should be removed on exit or error. By default, + * containers are removed on exit or error. This feature is provided for + * debugging purposes. + *
  • * */ @InterfaceAudience.Private @@ -145,6 +153,9 @@ @InterfaceAudience.Private public static final String ENV_DOCKER_CONTAINER_LOCAL_RESOURCE_MOUNTS = "YARN_CONTAINER_RUNTIME_DOCKER_LOCAL_RESOURCE_MOUNTS"; + @InterfaceAudience.Private + public static final String ENV_DOCKER_CONTAINER_KEEP_CONTAINER_ON_EXIT = + "YARN_CONTAINER_RUNTIME_DOCKER_KEEP_CONTAINER_ON_EXIT"; static final String CGROUPS_ROOT_DIRECTORY = "/sys/fs/cgroup"; @@ -155,6 +166,7 @@ private String defaultNetwork; private CGroupsHandler cGroupsHandler; private AccessControlList privilegedContainersAcl; + private DockerContainerStatusHandler dockerContainerStatusHandler; /** * Return whether the given environment variables indicate that the operation @@ -185,8 +197,8 @@ public static boolean isDockerContainerRequested( */ public DockerLinuxContainerRuntime(PrivilegedOperationExecutor privilegedOperationExecutor) { - this(privilegedOperationExecutor, - ResourceHandlerModule.getCGroupsHandler()); + this(privilegedOperationExecutor, ResourceHandlerModule + .getCGroupsHandler(), new DockerContainerStatusHandler()); } /** @@ -202,13 +214,38 @@ public DockerLinuxContainerRuntime(PrivilegedOperationExecutor public DockerLinuxContainerRuntime(PrivilegedOperationExecutor privilegedOperationExecutor, CGroupsHandler cGroupsHandler) { this.privilegedOperationExecutor = privilegedOperationExecutor; + this.dockerContainerStatusHandler = new DockerContainerStatusHandler(); + setCGroupHandler(cGroupsHandler); + } - if (cGroupsHandler == null) { + /** + * Create an instance using the given {@link PrivilegedOperationExecutor} + * instance for performing operations and the given {@link CGroupsHandler} + * instance. Inject a {@link DockerContainerStatusHandler} for testing. This + * constructor is intended for use in testing. + * + * @param privilegedOperationExecutor {@link PrivilegedOperationExecutor} + * instance. + * @param cGroupsHandler {@link CGroupsHandler} instance. + * @param dockerContainerStatusHandler {@link DockerContainerStatusHandler} + * instance. + */ + @VisibleForTesting + public DockerLinuxContainerRuntime(PrivilegedOperationExecutor + privilegedOperationExecutor, CGroupsHandler cGroupsHandler, + DockerContainerStatusHandler dockerContainerStatusHandler) { + this.privilegedOperationExecutor = privilegedOperationExecutor; + this.dockerContainerStatusHandler = dockerContainerStatusHandler; + setCGroupHandler(cGroupsHandler); + } + + private void setCGroupHandler(CGroupsHandler cGroupHandlerInstance) { + if (cGroupHandlerInstance == null) { if (LOG.isInfoEnabled()) { LOG.info("cGroupsHandler is null - cgroups not in use."); } } else { - this.cGroupsHandler = cGroupsHandler; + this.cGroupsHandler = cGroupHandlerInstance; } } @@ -411,6 +448,7 @@ public void launchContainer(ContainerRuntimeContext ctx) } validateContainerNetworkType(network); + validateKeepContainerOnExit(environment); if (imageName == null) { throw new ContainerExecutionException(ENV_DOCKER_CONTAINER_IMAGE @@ -550,45 +588,126 @@ public void launchContainer(ContainerRuntimeContext ctx) } } + /** + * Signal the docker container. + * + * Signals are used to check the liveliness of the container as well as to + * stop/kill the container. The following outlines the docker container + * signal handling. + * + *
      + *
    1. If the null signal is sent, run kill -0 on the pid. This is used + * to check if the container is still alive, which is necessary for + * reacquiring containers on NM restart.
    2. + *
    3. If any other signal is sent (SIGSTOP, SIGKILL, etc), attempt to + * stop and remove the docker container based on the following rules.
    4. + *
    5. If the docker container exists and is running, execute docker + * stop. This should not happen under normal circumstances given that + * container-executor runs docker wait, but this provides a safety net in + * case of failures related to docker wait.
    6. + *
    7. If {@code YARN_CONTAINER_RUNTIME_DOCKER_KEEP_CONTAINER_ON_EXIT} + * is false or not set, run docker rm on the docker container to remove + * it. If true, leave the container. This is useful for debugging.
    8. + *
    9. If the docker container is nonexistent or its status is unknown, + * do nothing.
    10. + *
    + * + * @param ctx the {@link ContainerRuntimeContext}. + * @throws ContainerExecutionException if the signaling fails. + */ @Override public void signalContainer(ContainerRuntimeContext ctx) throws ContainerExecutionException { - Container container = ctx.getContainer(); - ContainerExecutor.Signal signal = ctx.getExecutionAttribute(SIGNAL); - - PrivilegedOperation privOp = null; - // Handle liveliness checks, send null signal to pid - if(ContainerExecutor.Signal.NULL.equals(signal)) { - privOp = new PrivilegedOperation( - PrivilegedOperation.OperationType.SIGNAL_CONTAINER); - privOp.appendArgs(ctx.getExecutionAttribute(RUN_AS_USER), - ctx.getExecutionAttribute(USER), - Integer.toString(PrivilegedOperation.RunAsUserCommand - .SIGNAL_CONTAINER.getValue()), - ctx.getExecutionAttribute(PID), - Integer.toString(ctx.getExecutionAttribute(SIGNAL).getValue())); - - // All other signals handled as docker stop - } else { - String containerId = ctx.getContainer().getContainerId().toString(); - DockerStopCommand stopCommand = new DockerStopCommand(containerId); - String commandFile = dockerClient.writeCommandToTempFile(stopCommand, - containerId); - privOp = new PrivilegedOperation( - PrivilegedOperation.OperationType.RUN_DOCKER_CMD); - privOp.appendArgs(commandFile); + try { + if (ContainerExecutor.Signal.NULL + .equals(ctx.getExecutionAttribute(SIGNAL))) { + executeLivelinessCheck(ctx); + } else { + DockerContainerStatus containerStatus = dockerContainerStatusHandler + .getContainerStatus(ctx, conf, privilegedOperationExecutor); + if (!containerStatus.equals(DockerContainerStatus.NONEXISTENT) + && !containerStatus.equals(DockerContainerStatus.UNKNOWN)) { + if (containerStatus.equals(DockerContainerStatus.RUNNING)) { + DockerCommandExecutor + .executeDockerStop(ctx, conf, privilegedOperationExecutor); + } + if (!shouldKeepContainerOnExit(ctx)) { + DockerCommandExecutor + .executeDockerRm(ctx, conf, privilegedOperationExecutor); + } + } + } + } catch (ContainerExecutionException e) { + LOG.warn("Signal docker container failed. Exception: ", e); + throw new ContainerExecutionException("Signal docker container failed", + e.getExitCode(), e.getOutput(), e.getErrorOutput()); + } + } + + private Boolean shouldKeepContainerOnExit(ContainerRuntimeContext ctx) + throws ContainerExecutionException { + String containerId = ctx.getContainer().getContainerId().toString(); + if (conf. + getBoolean(YarnConfiguration.NM_DOCKER_ALLOW_KEEP_CONTAINER_ON_EXIT, + YarnConfiguration.DEFAULT_NM_DOCKER_ALLOW_KEEP_CONTAINER_ON_EXIT)) { + Map env = + ctx.getContainer().getLaunchContext().getEnvironment(); + if (env.containsKey(ENV_DOCKER_CONTAINER_KEEP_CONTAINER_ON_EXIT)) { + try { + validateKeepContainerOnExit(env); + } catch (ContainerExecutionException e) { + throw e; + } + if (env.get(ENV_DOCKER_CONTAINER_KEEP_CONTAINER_ON_EXIT) + .equalsIgnoreCase("true")) { + LOG.info("Docker container is not being removed due to user request. " + + "ContainerId: " + containerId); + return true; + } + } } + // Ready to remove the container + LOG.info( + "Docker container is ready for removal. ContainerId: " + containerId); + return false; + } - //Some failures here are acceptable. Let the calling executor decide. - privOp.disableFailureLogging(); + private void validateKeepContainerOnExit(Map env) + throws ContainerExecutionException { + if (conf. + getBoolean(YarnConfiguration.NM_DOCKER_ALLOW_KEEP_CONTAINER_ON_EXIT, + YarnConfiguration.DEFAULT_NM_DOCKER_ALLOW_KEEP_CONTAINER_ON_EXIT)) { + String keepContainer = + env.get(ENV_DOCKER_CONTAINER_KEEP_CONTAINER_ON_EXIT); + if (keepContainer != null && !keepContainer.equalsIgnoreCase("true") && + !keepContainer.equalsIgnoreCase("false")) { + String msg = "Only true and false are valid for" + + " YARN_CONTAINER_RUNTIME_DOCKER_KEEP_CONTAINER_ON_EXIT"; + throw new ContainerExecutionException(msg); + } + } + } + private void executeLivelinessCheck(ContainerRuntimeContext ctx) + throws ContainerExecutionException { + PrivilegedOperation signalOp = new PrivilegedOperation( + PrivilegedOperation.OperationType.SIGNAL_CONTAINER); + signalOp.appendArgs(ctx.getExecutionAttribute(RUN_AS_USER), + ctx.getExecutionAttribute(USER), Integer.toString( + PrivilegedOperation.RunAsUserCommand.SIGNAL_CONTAINER.getValue()), + ctx.getExecutionAttribute(PID), + Integer.toString(ctx.getExecutionAttribute(SIGNAL).getValue())); + signalOp.disableFailureLogging(); try { - privilegedOperationExecutor.executePrivilegedOperation(null, - privOp, null, container.getLaunchContext().getEnvironment(), - false, false); + privilegedOperationExecutor.executePrivilegedOperation(null, signalOp, + null, ctx.getContainer().getLaunchContext().getEnvironment(), false, + false); } catch (PrivilegedOperationException e) { - throw new ContainerExecutionException("Signal container failed", e - .getExitCode(), e.getOutput(), e.getErrorOutput()); + String msg = + "Liveliness check failed for PID: " + ctx.getExecutionAttribute(PID) + + ". Container may have already completed."; + throw new ContainerExecutionException(msg, e.getExitCode(), e.getOutput(), + e.getErrorOutput()); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerCommandExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerCommandExecutor.java new file mode 100644 index 0000000..bc745e3 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerCommandExecutor.java @@ -0,0 +1,131 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperation; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationException; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationExecutor; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerExecutionException; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerRuntimeContext; + +/** + * Utility class for executing common docker operations. + */ +public final class DockerCommandExecutor { + + private static final Log LOG = LogFactory.getLog(DockerCommandExecutor.class); + + private DockerCommandExecutor() { + } + + /** + * Execute a docker command and return the output. + * + * @param dockerCommand the docker command to run. + * @param ctx the container runtime context. + * @param conf the hadoop configuration. + * @param privilegedOperationExecutor the privileged operations executor. + * @return the output of the operation. + * @throws ContainerExecutionException if the operation fails. + */ + public static String executeDockerCommand(DockerCommand dockerCommand, + ContainerRuntimeContext ctx, Configuration conf, + PrivilegedOperationExecutor privilegedOperationExecutor) + throws ContainerExecutionException { + DockerClient dockerClient = new DockerClient(conf); + String containerId = ctx.getContainer().getContainerId().toString(); + String commandFile = + dockerClient.writeCommandToTempFile(dockerCommand, containerId); + PrivilegedOperation dockerOp = new PrivilegedOperation( + PrivilegedOperation.OperationType.RUN_DOCKER_CMD); + dockerOp.appendArgs(commandFile); + dockerOp.disableFailureLogging(); + try { + String result = privilegedOperationExecutor + .executePrivilegedOperation(null, dockerOp, null, + ctx.getContainer().getLaunchContext(). + getEnvironment(), true, false); + if (result != null && !result.isEmpty()) { + result = result.trim(); + } + return result; + } catch (PrivilegedOperationException e) { + throw new ContainerExecutionException("Docker operation failed", + e.getExitCode(), e.getOutput(), e.getErrorOutput()); + } + } + + /** + * Execute the docker rm command. + * + * @param ctx the container runtime context. + * @param conf the hadoop configuration. + * @param privilegedOperationExecutor the privileged operations executor. + * @throws ContainerExecutionException if the operation fails. + */ + public static void executeDockerRm(ContainerRuntimeContext ctx, + Configuration conf, + PrivilegedOperationExecutor privilegedOperationExecutor) + throws ContainerExecutionException { + String containerId = ctx.getContainer().getContainerId().toString(); + DockerRmCommand rmCommand = new DockerRmCommand(containerId); + LOG.debug("Running docker rm. ContainerId: " + containerId); + executeDockerCommand(rmCommand, ctx, conf, privilegedOperationExecutor); + } + + /** + * Execute the docker stop command. + * + * @param ctx the container runtime context. + * @param conf the hadoop configuration. + * @param privilegedOperationExecutor the privileged operations executor. + * @throws ContainerExecutionException if the operation fails. + */ + public static void executeDockerStop(ContainerRuntimeContext ctx, + Configuration conf, + PrivilegedOperationExecutor privilegedOperationExecutor) + throws ContainerExecutionException { + String containerId = ctx.getContainer().getContainerId().toString(); + DockerStopCommand stopCommand = new DockerStopCommand(containerId); + LOG.debug("Running docker stop. ContainerId: " + containerId); + executeDockerCommand(stopCommand, ctx, conf, privilegedOperationExecutor); + } + + /** + * Execute the docker inspect command. + * + * @param dockerInspectCommand the docker inspect command to execute. + * @param ctx the container runtime context. + * @param conf the hadoop configuration. + * @param privilegedOperationExecutor the privileged operations executor. + * @return the result of the docker inspect command. + * @throws ContainerExecutionException if the operation fails. + */ + public static String executeDockerInspect( + DockerInspectCommand dockerInspectCommand, ContainerRuntimeContext ctx, + Configuration conf, + PrivilegedOperationExecutor privilegedOperationExecutor) + throws ContainerExecutionException { + String containerId = ctx.getContainer().getContainerId().toString(); + LOG.debug("Running docker inspect. ContainerId: " + containerId); + return executeDockerCommand(dockerInspectCommand, ctx, conf, + privilegedOperationExecutor); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerContainerStatusHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerContainerStatusHandler.java new file mode 100644 index 0000000..0bccd94 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerContainerStatusHandler.java @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationExecutor; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerExecutionException; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerRuntimeContext; + +/** + * Encapsulate docker container status. + */ +public class DockerContainerStatusHandler { + + private static final Log LOG = + LogFactory.getLog(DockerContainerStatusHandler.class); + + /** + * Potential states that the docker status can return. + */ + public enum DockerContainerStatus { + CREATED("created"), + RUNNING("running"), + STOPPED("stopped"), + EXITED("exited"), + NONEXISTENT("nonexistent"), + UNKNOWN("unknown"); + + private final String name; + + DockerContainerStatus(String name) { + this.name = name; + } + + public String getName() { + return name; + } + } + + /** + * Get the status of the docker container. This runs a docker inspect to + * get the status. If the container no longer exists, docker inspect throws + * an exception and the nonexistent status is returned. + * + * @param ctx the container runtime context. + * @param conf the hadoop configuration. + * @param privilegedOperationExecutor the privileged operations executor. + * @return a {@link DockerContainerStatus} representing the current status. + */ + public DockerContainerStatus getContainerStatus(ContainerRuntimeContext ctx, + Configuration conf, + PrivilegedOperationExecutor privilegedOperationExecutor) { + String containerId = ctx.getContainer().getContainerId().toString(); + try { + DockerContainerStatus dockerContainerStatus; + String currentContainerStatus = + executeStatusCommand(ctx, conf, privilegedOperationExecutor); + if (currentContainerStatus == null) { + dockerContainerStatus = DockerContainerStatus.UNKNOWN; + } else if (currentContainerStatus + .equals(DockerContainerStatus.CREATED.getName())) { + dockerContainerStatus = DockerContainerStatus.CREATED; + } else if (currentContainerStatus + .equals(DockerContainerStatus.RUNNING.getName())) { + dockerContainerStatus = DockerContainerStatus.RUNNING; + } else if (currentContainerStatus + .equals(DockerContainerStatus.STOPPED.getName())) { + dockerContainerStatus = DockerContainerStatus.STOPPED; + } else if (currentContainerStatus + .equals(DockerContainerStatus.EXITED.getName())) { + dockerContainerStatus = DockerContainerStatus.EXITED; + } else { + dockerContainerStatus = DockerContainerStatus.UNKNOWN; + } + LOG.debug("Container Status: " + dockerContainerStatus.getName() + + " ContainerId: " + containerId); + return dockerContainerStatus; + } catch (ContainerExecutionException e) { + LOG.debug( + "Container Status: " + DockerContainerStatus.NONEXISTENT.getName() + + " ContainerId: " + containerId); + return DockerContainerStatus.NONEXISTENT; + } + } + + /** + * Execute the docker inspect command to retrieve the docker container's + * status. + * + * @param ctx the container runtime context. + * @param conf the hadoop configuration. + * @param privilegedOperationExecutor the privileged operations executor. + * @return the current container status. + * @throws ContainerExecutionException if the docker operation fails to run. + */ + protected String executeStatusCommand(ContainerRuntimeContext ctx, + Configuration conf, + PrivilegedOperationExecutor privilegedOperationExecutor) + throws ContainerExecutionException { + String containerId = ctx.getContainer().getContainerId().toString(); + DockerInspectCommand dockerInspectCommand = + new DockerInspectCommand(containerId).getContainerStatus(); + try { + return DockerCommandExecutor.executeDockerInspect(dockerInspectCommand, + ctx, conf, privilegedOperationExecutor); + } catch (ContainerExecutionException e) { + throw new ContainerExecutionException(e); + } + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/package-info.java new file mode 100644 index 0000000..189167c --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/package-info.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** + * Package containing classes related to Docker commands and common operations + * used within the @{link DockerLinuxContainerRuntime}. + */ +@InterfaceAudience.Private +@InterfaceStability.Unstable +package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c index ca3847e..b1836a4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c @@ -1286,7 +1286,6 @@ int launch_docker_container_as_user(const char * user, const char *app_id, char docker_wait_command[EXECUTOR_PATH_MAX]; char docker_logs_command[EXECUTOR_PATH_MAX]; char docker_inspect_command[EXECUTOR_PATH_MAX]; - char docker_rm_command[EXECUTOR_PATH_MAX]; int container_file_source =-1; int cred_file_source = -1; int BUFFER_SIZE = 4096; @@ -1437,19 +1436,6 @@ int launch_docker_container_as_user(const char * user, const char *app_id, } } - fprintf(LOGFILE, "Removing docker container post-exit...\n"); - snprintf(docker_rm_command, EXECUTOR_PATH_MAX, - "%s rm %s", docker_binary, container_id); - FILE* rm_docker = popen(docker_rm_command, "w"); - if (pclose (rm_docker) != 0) - { - fprintf (ERRORFILE, - "Could not remove container %s.\n", docker_rm_command); - fflush(ERRORFILE); - exit_code = UNABLE_TO_EXECUTE_CONTAINER_SCRIPT; - goto cleanup; - } - cleanup: //clean up docker command file unlink(command_file); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/privileged/MockPrivilegedOperationCaptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/privileged/MockPrivilegedOperationCaptor.java new file mode 100644 index 0000000..1d55330 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/privileged/MockPrivilegedOperationCaptor.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged; + +import java.io.File; +import java.util.List; +import java.util.Map; +import org.mockito.ArgumentCaptor; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyList; +import static org.mockito.Matchers.eq; +import org.mockito.Mockito; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +/** + * Captures operations from mock {@link PrivilegedOperation} instances. + */ +public final class MockPrivilegedOperationCaptor { + + private MockPrivilegedOperationCaptor() {} + + /** + * Capture the operation that should be performed by the + * PrivilegedOperationExecutor. + * + * @param mockExecutor mock PrivilegedOperationExecutor. + * @param invocationCount number of invocations expected. + * @return a list of operations that were invoked. + * @throws PrivilegedOperationException when the operation fails to execute. + */ + @SuppressWarnings("unchecked") + public static List capturePrivilegedOperations( + PrivilegedOperationExecutor mockExecutor, int invocationCount, + boolean grabOutput) throws PrivilegedOperationException { + ArgumentCaptor opCaptor = + ArgumentCaptor.forClass(PrivilegedOperation.class); + + //one or more invocations expected + //due to type erasure + mocking, this verification requires a suppress + // warning annotation on the entire method + verify(mockExecutor, times(invocationCount)) + .executePrivilegedOperation(anyList(), opCaptor.capture(), + any(File.class), any(Map.class), eq(grabOutput), eq(false)); + + //verification completed. we need to isolate specific invications. + // hence, reset mock here + Mockito.reset(mockExecutor); + + return opCaptor.getAllValues(); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerRuntimeTestingUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerRuntimeTestingUtils.java new file mode 100644 index 0000000..5028418 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerRuntimeTestingUtils.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime; + +import java.io.IOException; +import java.nio.charset.Charset; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperation; +import org.junit.Assert; + +/** + * Utility class for commonly used docker runtime testing helpers. + */ +public final class DockerRuntimeTestingUtils { + + private DockerRuntimeTestingUtils() { + } + + /** + * Ensure that the contents of the docker command file are correct and + * return the docker commands found in the command file. + * + * @param ops list of {@link PrivilegedOperation} from the operation + * invocation. + * @return the docker commands found in the docker command file. + * @throws IOException if the docker command file cannot be read. + */ + public static List getValidatedDockerCommands( + List ops) throws IOException { + try { + List dockerCommands = new ArrayList<>(); + for (PrivilegedOperation op : ops) { + Assert.assertEquals(op.getOperationType(), + PrivilegedOperation.OperationType.RUN_DOCKER_CMD); + String dockerCommandFile = op.getArguments().get(0); + List dockerCommandFileContents = Files + .readAllLines(Paths.get(dockerCommandFile), + Charset.forName("UTF-8")); + dockerCommands.addAll(dockerCommandFileContents); + } + return dockerCommands; + } catch (IOException e) { + throw new IOException("Unable to read the docker command file.", e); + } + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java index 3253394..234624e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java @@ -30,11 +30,14 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.MockPrivilegedOperationCaptor; import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperation; import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationException; import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationExecutor; import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.CGroupsHandler; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker.DockerContainerStatusHandler.DockerContainerStatus; import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker.DockerRunCommand; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker.MockDockerContainerStatusHandler; import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerExecutionException; import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerRuntimeConstants; import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerRuntimeContext; @@ -68,6 +71,7 @@ private Configuration conf; private PrivilegedOperationExecutor mockExecutor; private CGroupsHandler mockCGroupsHandler; + private MockDockerContainerStatusHandler dockerContainerStatusHandler; private String containerId; private Container container; private ContainerId cId; @@ -171,6 +175,10 @@ public void setup() { .setExecutionAttribute(CONTAINER_LOG_DIRS, containerLogDirs) .setExecutionAttribute(LOCALIZED_RESOURCES, localizedResources) .setExecutionAttribute(RESOURCES_OPTIONS, resourcesOptions); + + dockerContainerStatusHandler = new MockDockerContainerStatusHandler(); + conf.set(YarnConfiguration.NM_DOCKER_ALLOW_KEEP_CONTAINER_ON_EXIT, "true"); + } @Test @@ -810,16 +818,19 @@ public void testMountMultiple() public void testContainerLivelinessCheck() throws ContainerExecutionException, PrivilegedOperationException { - DockerLinuxContainerRuntime runtime = new DockerLinuxContainerRuntime( - mockExecutor, mockCGroupsHandler); + DockerLinuxContainerRuntime runtime = + new DockerLinuxContainerRuntime(mockExecutor, mockCGroupsHandler); builder.setExecutionAttribute(RUN_AS_USER, runAsUser) - .setExecutionAttribute(USER, user) - .setExecutionAttribute(PID, signalPid) + .setExecutionAttribute(USER, user).setExecutionAttribute(PID, signalPid) .setExecutionAttribute(SIGNAL, ContainerExecutor.Signal.NULL); - runtime.initialize(getConfigurationWithMockContainerExecutor()); + runtime.initialize( + TestDockerContainerRuntime.enableMockContainerExecutor(conf)); runtime.signalContainer(builder.build()); - PrivilegedOperation op = capturePrivilegedOperation(); + List ops = MockPrivilegedOperationCaptor + .capturePrivilegedOperations(mockExecutor, 1, false); + Assert.assertEquals(1, ops.size()); + PrivilegedOperation op = ops.get(0); Assert.assertEquals(op.getOperationType(), PrivilegedOperation.OperationType.SIGNAL_CONTAINER); Assert.assertEquals("run_as_user", op.getArguments().get(0)); @@ -833,57 +844,233 @@ public void testContainerLivelinessCheck() public void testDockerStopOnTermSignal() throws ContainerExecutionException, PrivilegedOperationException, IOException { - List dockerCommands = getDockerCommandsForSignal( - ContainerExecutor.Signal.TERM); - Assert.assertEquals(1, dockerCommands.size()); - Assert.assertEquals("stop container_id", dockerCommands.get(0)); + setKeepContainerEnv(true); + setDockerContainerStatus(DockerContainerStatus.RUNNING); + List dockerCommands = + getDockerCommandsForSignal(ContainerExecutor.Signal.TERM, 2); + Assert.assertEquals(2, dockerCommands.size()); + Assert.assertEquals("inspect --format='{{.State.Status}}' container_id", + dockerCommands.get(0)); + Assert.assertEquals("stop container_id", dockerCommands.get(1)); } @Test public void testDockerStopOnKillSignal() throws ContainerExecutionException, PrivilegedOperationException, IOException { - List dockerCommands = getDockerCommandsForSignal( - ContainerExecutor.Signal.KILL); - Assert.assertEquals(1, dockerCommands.size()); - Assert.assertEquals("stop container_id", dockerCommands.get(0)); + setKeepContainerEnv(true); + setDockerContainerStatus(DockerContainerStatus.RUNNING); + List dockerCommands = + getDockerCommandsForSignal(ContainerExecutor.Signal.KILL, 2); + Assert.assertEquals(2, dockerCommands.size()); + Assert.assertEquals("inspect --format='{{.State.Status}}' container_id", + dockerCommands.get(0)); + Assert.assertEquals("stop container_id", dockerCommands.get(1)); } @Test public void testDockerStopOnQuitSignal() throws ContainerExecutionException, PrivilegedOperationException, IOException { - List dockerCommands = getDockerCommandsForSignal( - ContainerExecutor.Signal.QUIT); + setKeepContainerEnv(true); + setDockerContainerStatus(DockerContainerStatus.RUNNING); + List dockerCommands = + getDockerCommandsForSignal(ContainerExecutor.Signal.QUIT, 2); + Assert.assertEquals(2, dockerCommands.size()); + Assert.assertEquals("inspect --format='{{.State.Status}}' container_id", + dockerCommands.get(0)); + Assert.assertEquals("stop container_id", dockerCommands.get(1)); + } + + @Test + public void testDockerStopRmOnTermSignal() + throws ContainerExecutionException, PrivilegedOperationException, + IOException { + setKeepContainerEnv(false); + setDockerContainerStatus(DockerContainerStatus.RUNNING); + List dockerCommands = + getDockerCommandsForSignal(ContainerExecutor.Signal.TERM, 3); + Assert.assertEquals(3, dockerCommands.size()); + Assert.assertEquals("inspect --format='{{.State.Status}}' container_id", + dockerCommands.get(0)); + Assert.assertEquals("stop container_id", dockerCommands.get(1)); + Assert.assertEquals("rm container_id", dockerCommands.get(2)); + } + + @Test + public void testDockerStopRmOnKillSignal() + throws ContainerExecutionException, PrivilegedOperationException, + IOException { + setKeepContainerEnv(false); + setDockerContainerStatus(DockerContainerStatus.RUNNING); + List dockerCommands = + getDockerCommandsForSignal(ContainerExecutor.Signal.KILL, 3); + Assert.assertEquals(3, dockerCommands.size()); + Assert.assertEquals("inspect --format='{{.State.Status}}' container_id", + dockerCommands.get(0)); + Assert.assertEquals("stop container_id", dockerCommands.get(1)); + Assert.assertEquals("rm container_id", dockerCommands.get(2)); + } + + @Test + public void testDockerStopRmOnQuitSignal() + throws ContainerExecutionException, PrivilegedOperationException, + IOException { + setKeepContainerEnv(false); + setDockerContainerStatus(DockerContainerStatus.RUNNING); + List dockerCommands = + getDockerCommandsForSignal(ContainerExecutor.Signal.QUIT, 3); + Assert.assertEquals(3, dockerCommands.size()); + Assert.assertEquals("inspect --format='{{.State.Status}}' container_id", + dockerCommands.get(0)); + Assert.assertEquals("stop container_id", dockerCommands.get(1)); + Assert.assertEquals("rm container_id", dockerCommands.get(2)); + } + + @Test + public void testDockerRmCreatedStatus() + throws ContainerExecutionException, PrivilegedOperationException, + IOException { + setKeepContainerEnv(false); + setDockerContainerStatus(DockerContainerStatus.CREATED); + List dockerCommands = + getDockerCommandsForSignal(ContainerExecutor.Signal.QUIT, 2); + Assert.assertEquals(2, dockerCommands.size()); + Assert.assertEquals("inspect --format='{{.State.Status}}' container_id", + dockerCommands.get(0)); + Assert.assertEquals("rm container_id", dockerCommands.get(1)); + } + + @Test + public void testDockerRmStoppedStatus() + throws ContainerExecutionException, PrivilegedOperationException, + IOException { + setKeepContainerEnv(false); + setDockerContainerStatus(DockerContainerStatus.STOPPED); + List dockerCommands = + getDockerCommandsForSignal(ContainerExecutor.Signal.QUIT, 2); + Assert.assertEquals(2, dockerCommands.size()); + Assert.assertEquals("inspect --format='{{.State.Status}}' container_id", + dockerCommands.get(0)); + Assert.assertEquals("rm container_id", dockerCommands.get(1)); + } + + @Test + public void testDockerRmExitedStatus() + throws ContainerExecutionException, PrivilegedOperationException, + IOException { + setKeepContainerEnv(false); + setDockerContainerStatus(DockerContainerStatus.EXITED); + List dockerCommands = + getDockerCommandsForSignal(ContainerExecutor.Signal.QUIT, 2); + Assert.assertEquals(2, dockerCommands.size()); + Assert.assertEquals("inspect --format='{{.State.Status}}' container_id", + dockerCommands.get(0)); + Assert.assertEquals("rm container_id", dockerCommands.get(1)); + } + + @Test + public void testDockerNoopForNonexistentContainer() + throws ContainerExecutionException, PrivilegedOperationException, + IOException { + setKeepContainerEnv(false); + setDockerContainerStatus(DockerContainerStatus.NONEXISTENT); + List dockerCommands = + getDockerCommandsForSignal(ContainerExecutor.Signal.QUIT, 1); Assert.assertEquals(1, dockerCommands.size()); - Assert.assertEquals("stop container_id", dockerCommands.get(0)); + Assert.assertEquals("inspect --format='{{.State.Status}}' container_id", + dockerCommands.get(0)); + } + + @Test + public void testDockerNoopForUnknownContainer() + throws ContainerExecutionException, PrivilegedOperationException, + IOException { + setKeepContainerEnv(false); + setDockerContainerStatus(DockerContainerStatus.UNKNOWN); + List dockerCommands = + getDockerCommandsForSignal(ContainerExecutor.Signal.QUIT, 1); + Assert.assertEquals(1, dockerCommands.size()); + Assert.assertEquals("inspect --format='{{.State.Status}}' container_id", + dockerCommands.get(0)); + } + + @Test + public void testDockerStopRmWhenKeepContainerYarnConfIsFalse() + throws ContainerExecutionException, PrivilegedOperationException, + IOException { + conf.set(YarnConfiguration.NM_DOCKER_ALLOW_KEEP_CONTAINER_ON_EXIT, "false"); + setKeepContainerEnv(true); + setDockerContainerStatus(DockerContainerStatus.RUNNING); + List dockerCommands = + getDockerCommandsForSignal(ContainerExecutor.Signal.QUIT, 3); + Assert.assertEquals(3, dockerCommands.size()); + Assert.assertEquals("inspect --format='{{.State.Status}}' container_id", + dockerCommands.get(0)); + Assert.assertEquals("stop container_id", dockerCommands.get(1)); + Assert.assertEquals("rm container_id", dockerCommands.get(2)); } private List getDockerCommandsForSignal( - ContainerExecutor.Signal signal) + ContainerExecutor.Signal signal, int expectedCommandCount) throws ContainerExecutionException, PrivilegedOperationException, IOException { + DockerLinuxContainerRuntime runtime = getRuntimeForSignal(signal); + initializeMockContainerExecutor(runtime); + setExecutionAttributesForSignal(signal); + runtime.signalContainer(builder.build()); + List ops = MockPrivilegedOperationCaptor + .capturePrivilegedOperations(mockExecutor, expectedCommandCount, true); + return DockerRuntimeTestingUtils.getValidatedDockerCommands(ops); + } - DockerLinuxContainerRuntime runtime = new DockerLinuxContainerRuntime( - mockExecutor, mockCGroupsHandler); + private DockerLinuxContainerRuntime getRuntimeForSignal( + ContainerExecutor.Signal signal) + throws ContainerExecutionException { + DockerLinuxContainerRuntime runtime = + new DockerLinuxContainerRuntime(mockExecutor, mockCGroupsHandler, + dockerContainerStatusHandler); builder.setExecutionAttribute(RUN_AS_USER, runAsUser) - .setExecutionAttribute(USER, user) - .setExecutionAttribute(PID, signalPid) + .setExecutionAttribute(USER, user).setExecutionAttribute(PID, signalPid) .setExecutionAttribute(SIGNAL, signal); - runtime.initialize(getConfigurationWithMockContainerExecutor()); - runtime.signalContainer(builder.build()); + initializeMockContainerExecutor(runtime); + return runtime; + } - PrivilegedOperation op = capturePrivilegedOperation(); - Assert.assertEquals(op.getOperationType(), - PrivilegedOperation.OperationType.RUN_DOCKER_CMD); - String dockerCommandFile = op.getArguments().get(0); - return Files.readAllLines(Paths.get(dockerCommandFile), - Charset.forName("UTF-8")); + private void setDockerContainerStatus(DockerContainerStatus status) { + dockerContainerStatusHandler.setDockerContainerStatus(status); + } + + private void initializeMockContainerExecutor( + DockerLinuxContainerRuntime runtime) throws ContainerExecutionException { + runtime.initialize( + TestDockerContainerRuntime.enableMockContainerExecutor(conf)); + } + + private void setExecutionAttributesForSignal( + ContainerExecutor.Signal signal) { + builder.setExecutionAttribute(RUN_AS_USER, runAsUser) + .setExecutionAttribute(USER, user).setExecutionAttribute(PID, signalPid) + .setExecutionAttribute(SIGNAL, signal); + } + + private void setKeepContainerEnv(boolean keepContainer) { + env.put( + DockerLinuxContainerRuntime.ENV_DOCKER_CONTAINER_KEEP_CONTAINER_ON_EXIT, + String.valueOf(keepContainer)); } - private Configuration getConfigurationWithMockContainerExecutor() { + /** + * Return a configuration object with the mock container executor binary + * preconfigured. + * + * @param conf The hadoop configuration. + * @return The hadoop configuration. + */ + public static Configuration enableMockContainerExecutor( + Configuration conf) { File f = new File("./src/test/resources/mock-container-executor"); - if(!FileUtil.canExecute(f)) { + if (!FileUtil.canExecute(f)) { FileUtil.setExecutable(f, true); } String executorPath = f.getAbsolutePath(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/MockDockerContainerStatusHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/MockDockerContainerStatusHandler.java new file mode 100644 index 0000000..ecb089a --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/MockDockerContainerStatusHandler.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker; + +import com.google.common.annotations.VisibleForTesting; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationExecutor; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerExecutionException; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerRuntimeContext; + +/** + * Mock container status handler. + */ +public class MockDockerContainerStatusHandler extends + DockerContainerStatusHandler { + + private static final Log LOG = + LogFactory.getLog(MockDockerContainerStatusHandler.class); + + private DockerContainerStatus status; + + /** + * Set the container status. This is intended for testing only. + * + * @param containerStatus the status state to inject. + */ + @VisibleForTesting + public void setDockerContainerStatus(DockerContainerStatus containerStatus) { + status = containerStatus; + } + + /** + * Reset the container status so that previous state does not impact future + * tests. + */ + @VisibleForTesting + public void resetDockerContainerStatus() { + status = null; + } + + /** + * Execute the docker inspect command to retrieve the docker container's + * status. + * + * @param ctx the container runtime context. + * @param conf the hadoop configuration. + * @param privilegedOperationExecutor the privileged operations executor. + * @return a string representing the current container status. + * @throws ContainerExecutionException if the docker operation fails to run. + */ + @Override + protected String executeStatusCommand(ContainerRuntimeContext ctx, + Configuration conf, + PrivilegedOperationExecutor privilegedOperationExecutor) + throws ContainerExecutionException { + + String containerStatus; + String containerId = ctx.getContainer().getContainerId().toString(); + DockerInspectCommand dockerInspectCommand = + new DockerInspectCommand(containerId).getContainerStatus(); + try { + containerStatus = DockerCommandExecutor + .executeDockerInspect(dockerInspectCommand, ctx, conf, + privilegedOperationExecutor); + LOG.debug("Container Status: " + containerStatus + " ContainerId: " + + containerId); + } catch (ContainerExecutionException e) { + throw new ContainerExecutionException( + "Container Status: " + DockerContainerStatus.NONEXISTENT.getName() + + "ContainerId: " + containerId); + } + + // Allow for injecting the container's status for testing. + if (status != null) { + containerStatus = status.getName(); + } + + return containerStatus; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerCommandExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerCommandExecutor.java new file mode 100644 index 0000000..6d60fed --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerCommandExecutor.java @@ -0,0 +1,158 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker; + +import java.util.HashMap; +import java.util.List; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.MockPrivilegedOperationCaptor; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperation; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationExecutor; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.CGroupsHandler; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.DockerLinuxContainerRuntime; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.DockerRuntimeTestingUtils; +import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.CONTAINER_ID_STR; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.TestDockerContainerRuntime; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerRuntimeContext; +import static org.junit.Assert.assertEquals; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mockito; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * Test common docker commands. + */ +public class TestDockerCommandExecutor { + + private static final String MOCK_CONTAINER_ID = "container_id"; + + private PrivilegedOperationExecutor mockExecutor; + private CGroupsHandler mockCGroupsHandler; + private Configuration configuration; + private ContainerRuntimeContext.Builder builder; + private DockerLinuxContainerRuntime runtime; + private Container container; + private ContainerId cId; + private ContainerLaunchContext context; + private HashMap env; + + @Before + public void setUp() throws Exception { + mockExecutor = Mockito.mock(PrivilegedOperationExecutor.class); + mockCGroupsHandler = Mockito.mock(CGroupsHandler.class); + configuration = new Configuration(); + runtime = new DockerLinuxContainerRuntime(mockExecutor, mockCGroupsHandler); + container = mock(Container.class); + cId = mock(ContainerId.class); + context = mock(ContainerLaunchContext.class); + env = new HashMap<>(); + builder = new ContainerRuntimeContext.Builder(container); + + when(container.getContainerId()).thenReturn(cId); + when(cId.toString()).thenReturn(MOCK_CONTAINER_ID); + when(container.getLaunchContext()).thenReturn(context); + when(context.getEnvironment()).thenReturn(env); + + builder.setExecutionAttribute(CONTAINER_ID_STR, MOCK_CONTAINER_ID); + runtime.initialize( + TestDockerContainerRuntime.enableMockContainerExecutor(configuration)); + } + + @Test + public void testExecuteDockerCommand() throws Exception { + DockerStopCommand dockerStopCommand = + new DockerStopCommand(MOCK_CONTAINER_ID); + DockerCommandExecutor + .executeDockerCommand(dockerStopCommand, builder.build(), configuration, + mockExecutor); + List ops = MockPrivilegedOperationCaptor + .capturePrivilegedOperations(mockExecutor, 1, true); + assertEquals(1, ops.size()); + assertEquals(PrivilegedOperation.OperationType.RUN_DOCKER_CMD.name(), + ops.get(0).getOperationType().name()); + } + + @Test + public void testExecuteDockerRm() throws Exception { + DockerCommandExecutor + .executeDockerRm(builder.build(), configuration, mockExecutor); + List ops = MockPrivilegedOperationCaptor + .capturePrivilegedOperations(mockExecutor, 1, true); + List dockerCommands = + DockerRuntimeTestingUtils.getValidatedDockerCommands(ops); + assertEquals(1, ops.size()); + assertEquals(PrivilegedOperation.OperationType.RUN_DOCKER_CMD.name(), + ops.get(0).getOperationType().name()); + assertEquals(1, dockerCommands.size()); + assertEquals("rm " + MOCK_CONTAINER_ID, dockerCommands.get(0)); + } + + @Test + public void testExecuteDockerStop() throws Exception { + DockerCommandExecutor + .executeDockerStop(builder.build(), configuration, mockExecutor); + List ops = MockPrivilegedOperationCaptor + .capturePrivilegedOperations(mockExecutor, 1, true); + List dockerCommands = + DockerRuntimeTestingUtils.getValidatedDockerCommands(ops); + assertEquals(1, ops.size()); + assertEquals(PrivilegedOperation.OperationType.RUN_DOCKER_CMD.name(), + ops.get(0).getOperationType().name()); + assertEquals(1, dockerCommands.size()); + assertEquals("stop " + MOCK_CONTAINER_ID, dockerCommands.get(0)); + } + + @Test + public void testExecuteDockerInspect() throws Exception { + DockerCommandExecutor + .executeDockerInspect(new DockerInspectCommand(MOCK_CONTAINER_ID), + builder.build(), configuration, mockExecutor); + List ops = MockPrivilegedOperationCaptor + .capturePrivilegedOperations(mockExecutor, 1, true); + List dockerCommands = + DockerRuntimeTestingUtils.getValidatedDockerCommands(ops); + assertEquals(1, ops.size()); + assertEquals(PrivilegedOperation.OperationType.RUN_DOCKER_CMD.name(), + ops.get(0).getOperationType().name()); + assertEquals(1, dockerCommands.size()); + assertEquals("inspect", dockerCommands.get(0)); + } + + @Test + public void testExecuteDockerInspectStatus() throws Exception { + DockerInspectCommand dockerInspectCommand = + new DockerInspectCommand(MOCK_CONTAINER_ID).getContainerStatus(); + DockerCommandExecutor + .executeDockerInspect(dockerInspectCommand, builder.build(), + configuration, mockExecutor); + List ops = MockPrivilegedOperationCaptor + .capturePrivilegedOperations(mockExecutor, 1, true); + List dockerCommands = + DockerRuntimeTestingUtils.getValidatedDockerCommands(ops); + assertEquals(1, ops.size()); + assertEquals(PrivilegedOperation.OperationType.RUN_DOCKER_CMD.name(), + ops.get(0).getOperationType().name()); + assertEquals(1, dockerCommands.size()); + assertEquals("inspect --format='{{.State.Status}}' " + MOCK_CONTAINER_ID, + dockerCommands.get(0)); + } +} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerContainerStatusHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerContainerStatusHandler.java new file mode 100644 index 0000000..c9f9d3f --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerContainerStatusHandler.java @@ -0,0 +1,138 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker; + +import java.util.HashMap; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationExecutor; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.CGroupsHandler; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.DockerLinuxContainerRuntime; +import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.CONTAINER_ID_STR; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.TestDockerContainerRuntime; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerRuntimeContext; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mockito; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * Test operations for the {@link DockerContainerStatusHandler}. + */ +public class TestDockerContainerStatusHandler { + + private static final String MOCK_CONTAINER_ID = "container_id"; + + private PrivilegedOperationExecutor mockExecutor; + private CGroupsHandler mockCGroupsHandler; + private MockDockerContainerStatusHandler mockDockerContainerStatusHandler; + private Configuration configuration; + private ContainerRuntimeContext.Builder builder; + private DockerLinuxContainerRuntime runtime; + private Container container; + private ContainerId cId; + private ContainerLaunchContext context; + private HashMap env; + + @Before + public void setUp() throws Exception { + mockExecutor = Mockito.mock(PrivilegedOperationExecutor.class); + mockCGroupsHandler = Mockito.mock(CGroupsHandler.class); + mockDockerContainerStatusHandler = new MockDockerContainerStatusHandler(); + configuration = new Configuration(); + runtime = new DockerLinuxContainerRuntime(mockExecutor, mockCGroupsHandler, + mockDockerContainerStatusHandler); + container = mock(Container.class); + cId = mock(ContainerId.class); + context = mock(ContainerLaunchContext.class); + env = new HashMap<>(); + builder = new ContainerRuntimeContext.Builder(container); + + when(container.getContainerId()).thenReturn(cId); + when(cId.toString()).thenReturn(MOCK_CONTAINER_ID); + when(container.getLaunchContext()).thenReturn(context); + when(context.getEnvironment()).thenReturn(env); + + builder.setExecutionAttribute(CONTAINER_ID_STR, MOCK_CONTAINER_ID); + runtime.initialize( + TestDockerContainerRuntime.enableMockContainerExecutor(configuration)); + } + + @Test + public void testSetDockerContainerStatusState() throws Exception { + mockDockerContainerStatusHandler.setDockerContainerStatus( + DockerContainerStatusHandler.DockerContainerStatus.RUNNING); + Assert.assertEquals( + DockerContainerStatusHandler.DockerContainerStatus.RUNNING, + mockDockerContainerStatusHandler + .getContainerStatus(builder.build(), configuration, mockExecutor)); + } + + @Test + public void testResetDockerContainerStatusState() throws Exception { + mockDockerContainerStatusHandler.resetDockerContainerStatus(); + Assert.assertEquals( + DockerContainerStatusHandler.DockerContainerStatus.UNKNOWN, + mockDockerContainerStatusHandler + .getContainerStatus(builder.build(), configuration, mockExecutor)); + } + + @Test + public void testGetContainerStatusForRunning() throws Exception { + mockDockerContainerStatusHandler.setDockerContainerStatus( + DockerContainerStatusHandler.DockerContainerStatus.RUNNING); + Assert.assertEquals( + DockerContainerStatusHandler.DockerContainerStatus.RUNNING, + mockDockerContainerStatusHandler + .getContainerStatus(builder.build(), configuration, mockExecutor)); + } + + @Test + public void testGetContainerStatusForStopped() throws Exception { + mockDockerContainerStatusHandler.setDockerContainerStatus( + DockerContainerStatusHandler.DockerContainerStatus.STOPPED); + Assert.assertEquals( + DockerContainerStatusHandler.DockerContainerStatus.STOPPED, + mockDockerContainerStatusHandler + .getContainerStatus(builder.build(), configuration, mockExecutor)); + } + + @Test + public void testGetContainerStatusForExited() throws Exception { + mockDockerContainerStatusHandler.setDockerContainerStatus( + DockerContainerStatusHandler.DockerContainerStatus.EXITED); + Assert + .assertEquals(DockerContainerStatusHandler.DockerContainerStatus.EXITED, + mockDockerContainerStatusHandler + .getContainerStatus(builder.build(), configuration, + mockExecutor)); + } + + @Test + public void testGetContainerStatusForUnknown() throws Exception { + mockDockerContainerStatusHandler.setDockerContainerStatus( + DockerContainerStatusHandler.DockerContainerStatus.UNKNOWN); + Assert.assertEquals( + DockerContainerStatusHandler.DockerContainerStatus.UNKNOWN, + mockDockerContainerStatusHandler + .getContainerStatus(builder.build(), configuration, mockExecutor)); + } +} \ No newline at end of file