diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java index 33b8add..930c605 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java @@ -40,9 +40,10 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.CGroupsHandler; import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerModule; import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker.DockerClient; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker.DockerCommandExecutor; import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker.DockerInspectCommand; import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker.DockerRunCommand; -import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker.DockerStopCommand; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker.DockerContainerStatusHandler; import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerExecutionException; import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerRuntime; import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerRuntimeConstants; @@ -120,6 +121,12 @@ * source is an absolute path that is not a symlink and that points to a * localized resource. * + *
  • + * {@code YARN_CONTAINER_RUNTIME_DOCKER_KEEP_CONTAINER_ON_EXIT} controls + * whether a container should be removed on exit or error. By default, + * containers are removed on exit or error. This feature is provided for + * debugging purposes. + *
  • * */ @InterfaceAudience.Private @@ -145,6 +152,9 @@ @InterfaceAudience.Private public static final String ENV_DOCKER_CONTAINER_LOCAL_RESOURCE_MOUNTS = "YARN_CONTAINER_RUNTIME_DOCKER_LOCAL_RESOURCE_MOUNTS"; + @InterfaceAudience.Private + public static final String ENV_DOCKER_CONTAINER_KEEP_CONTAINER_ON_EXIT = + "YARN_CONTAINER_RUNTIME_DOCKER_KEEP_CONTAINER_ON_EXIT"; private Configuration conf; private DockerClient dockerClient; @@ -153,6 +163,7 @@ private String defaultNetwork; private CGroupsHandler cGroupsHandler; private AccessControlList privilegedContainersAcl; + private DockerContainerStatusHandler dockerContainerStatusHandler; /** * Return whether the given environment variables indicate that the operation @@ -183,8 +194,8 @@ public static boolean isDockerContainerRequested( */ public DockerLinuxContainerRuntime(PrivilegedOperationExecutor privilegedOperationExecutor) { - this(privilegedOperationExecutor, - ResourceHandlerModule.getCGroupsHandler()); + this(privilegedOperationExecutor, ResourceHandlerModule + .getCGroupsHandler(), new DockerContainerStatusHandler()); } /** @@ -200,13 +211,38 @@ public DockerLinuxContainerRuntime(PrivilegedOperationExecutor public DockerLinuxContainerRuntime(PrivilegedOperationExecutor privilegedOperationExecutor, CGroupsHandler cGroupsHandler) { this.privilegedOperationExecutor = privilegedOperationExecutor; + this.dockerContainerStatusHandler = new DockerContainerStatusHandler(); + setCGroupHandler(cGroupsHandler); + } - if (cGroupsHandler == null) { + /** + * Create an instance using the given {@link PrivilegedOperationExecutor} + * instance for performing operations and the given {@link CGroupsHandler} + * instance. Inject a {@link DockerContainerStatusHandler} for testing. This + * constructor is intended for use in testing. + * + * @param privilegedOperationExecutor the {@link PrivilegedOperationExecutor} + * instance + * @param cGroupsHandler the {@link CGroupsHandler} instance + * @param dockerContainerStatusHandler the + * {@link DockerContainerStatusHandler} instance + */ + @VisibleForTesting + public DockerLinuxContainerRuntime(PrivilegedOperationExecutor + privilegedOperationExecutor, CGroupsHandler cGroupsHandler, + DockerContainerStatusHandler dockerContainerStatusHandler) { + this.privilegedOperationExecutor = privilegedOperationExecutor; + this.dockerContainerStatusHandler = dockerContainerStatusHandler; + setCGroupHandler(cGroupsHandler); + } + + private void setCGroupHandler(CGroupsHandler cGroupHandlerInstance) { + if (cGroupHandlerInstance == null) { if (LOG.isInfoEnabled()) { LOG.info("cGroupsHandler is null - cgroups not in use."); } } else { - this.cGroupsHandler = cGroupsHandler; + this.cGroupsHandler = cGroupHandlerInstance; } } @@ -548,45 +584,116 @@ public void launchContainer(ContainerRuntimeContext ctx) } } + /** + * Signal the docker container. + * + * Signals are used to check the liveliness of the container as well as to + * stop/kill the container. The following outlines the docker container + * signal handling. + * + *
      + *
    1. If the null signal is sent, run kill -0 on the pid. This is used + * to check if the container is still alive, which is necessary for + * reacquiring containers on NM restart.
    2. + *
    3. If any other signal is sent (SIGSTOP, SIGKILL, etc), attempt to + * stop and remove it based on the following rules.
    4. + *
    5. If the docker container exists and is running, execute docker + * stop. This should not happen under normal circumstances given that + * container-executor runs docker wait, but this provides a safety net in + * case of failures related to docker wait.
    6. + *
    7. If {@code YARN_CONTAINER_RUNTIME_DOCKER_KEEP_CONTAINER_ON_EXIT} + * is false or not set, run docker rm on the docker container to clean + * it up. If true, leave the container. This is useful for debugging.
    8. + *
    9. If the docker container is nonexistent or its status is unknown, + * do nothing.
    10. + *
    + * + * @param ctx the {@link ContainerRuntimeContext}. + * @throws ContainerExecutionException if the signaling fails. + */ @Override public void signalContainer(ContainerRuntimeContext ctx) throws ContainerExecutionException { - Container container = ctx.getContainer(); - ContainerExecutor.Signal signal = ctx.getExecutionAttribute(SIGNAL); - - PrivilegedOperation privOp = null; - // Handle liveliness checks, send null signal to pid - if(ContainerExecutor.Signal.NULL.equals(signal)) { - privOp = new PrivilegedOperation( - PrivilegedOperation.OperationType.SIGNAL_CONTAINER); - privOp.appendArgs(ctx.getExecutionAttribute(RUN_AS_USER), - ctx.getExecutionAttribute(USER), - Integer.toString(PrivilegedOperation.RunAsUserCommand - .SIGNAL_CONTAINER.getValue()), - ctx.getExecutionAttribute(PID), - Integer.toString(ctx.getExecutionAttribute(SIGNAL).getValue())); - - // All other signals handled as docker stop - } else { - String containerId = ctx.getContainer().getContainerId().toString(); - DockerStopCommand stopCommand = new DockerStopCommand(containerId); - String commandFile = dockerClient.writeCommandToTempFile(stopCommand, - containerId); - privOp = new PrivilegedOperation( - PrivilegedOperation.OperationType.RUN_DOCKER_CMD); - privOp.appendArgs(commandFile); + try { + if (ContainerExecutor.Signal.NULL + .equals(ctx.getExecutionAttribute(SIGNAL))) { + executeLivelinessCheck(ctx); + } else { + DockerContainerStatusHandler.StatusState containerStatus = + dockerContainerStatusHandler + .getContainerStatus(ctx, conf, privilegedOperationExecutor); + if (!containerStatus + .equals(DockerContainerStatusHandler.StatusState.NONEXISTENT) + && !containerStatus + .equals(DockerContainerStatusHandler.StatusState.UNKNOWN)) { + if (containerStatus + .equals(DockerContainerStatusHandler.StatusState.RUNNING)) { + DockerCommandExecutor + .executeDockerStop(ctx, conf, privilegedOperationExecutor); + } + if (!shouldKeepContainerOnExit(ctx)) { + DockerCommandExecutor + .executeDockerRm(ctx, conf, privilegedOperationExecutor); + } + } + } + } catch (ContainerExecutionException e) { + LOG.warn("Signal docker container failed. Exception: ", e); + throw new ContainerExecutionException("Signal docker container failed", + e.getExitCode(), e.getOutput(), e.getErrorOutput()); } + } - //Some failures here are acceptable. Let the calling executor decide. - privOp.disableFailureLogging(); + private boolean shouldKeepContainerOnExit(ContainerRuntimeContext ctx) { + Map env = + ctx.getContainer().getLaunchContext().getEnvironment(); + String containerId = ctx.getContainer().getContainerId().toString(); + if (env.containsKey(ENV_DOCKER_CONTAINER_KEEP_CONTAINER_ON_EXIT)) { + String keepContainer = + env.get(ENV_DOCKER_CONTAINER_KEEP_CONTAINER_ON_EXIT); + + // Validate the configured value + if (!keepContainer.equalsIgnoreCase("true") && + !keepContainer.equalsIgnoreCase("false")) { + throw new IllegalArgumentException("Only true and false are valid for" + + " YARN_CONTAINER_RUNTIME_DOCKER_KEEP_CONTAINER_ON_EXIT"); + } + // Keep the container + if (keepContainer.equalsIgnoreCase("true")) { + LOG.debug("Docker container is not being removed due to user request. " + + "ContainerId: " + containerId); + return true; + } + } + // Ready to remove the container + LOG.debug("Docker container ready for removal. ContainerId: " + + containerId); + return false; + } + + private void executeLivelinessCheck(ContainerRuntimeContext ctx) + throws ContainerExecutionException { + PrivilegedOperation signalOp = new PrivilegedOperation( + PrivilegedOperation.OperationType.SIGNAL_CONTAINER); + signalOp.appendArgs(ctx.getExecutionAttribute(RUN_AS_USER), + ctx.getExecutionAttribute(USER), Integer.toString( + PrivilegedOperation.RunAsUserCommand.SIGNAL_CONTAINER.getValue()), + ctx.getExecutionAttribute(PID), + Integer.toString(ctx.getExecutionAttribute(SIGNAL).getValue())); + signalOp.disableFailureLogging(); try { - privilegedOperationExecutor.executePrivilegedOperation(null, - privOp, null, container.getLaunchContext().getEnvironment(), - false, false); + privilegedOperationExecutor + .executePrivilegedOperation(null, signalOp, null, + ctx.getContainer().getLaunchContext(). + getEnvironment(), false, false); } catch (PrivilegedOperationException e) { - throw new ContainerExecutionException("Signal container failed", e - .getExitCode(), e.getOutput(), e.getErrorOutput()); + String msg = + "Liveliness check failed for PID: " + ctx.getExecutionAttribute(PID) + + ". Container may have already completed."; + LOG.warn(msg); + throw new ContainerExecutionException(msg, e.getExitCode(), e.getOutput(), + e.getErrorOutput()); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerCommandExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerCommandExecutor.java new file mode 100644 index 0000000..7bb66d6 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerCommandExecutor.java @@ -0,0 +1,127 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperation; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationException; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationExecutor; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerExecutionException; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerRuntimeContext; + +/** + * Utility class for executing common docker operations. + */ +public final class DockerCommandExecutor { + + private static final Log LOG = LogFactory.getLog(DockerCommandExecutor.class); + + private DockerCommandExecutor() { + } + + /** + * Execute a docker command and return the output. + * + * @param dockerCommand the docker command to run + * @param ctx the container runtime context. + * @param conf the hadoop configuration. + * @param privilegedOperationExecutor the privileged operations executor. + * @return the output of the operation. + * @throws ContainerExecutionException if the operation fails. + */ + public static String executeDockerCommand(DockerCommand dockerCommand, + ContainerRuntimeContext ctx, Configuration conf, + PrivilegedOperationExecutor privilegedOperationExecutor) + throws ContainerExecutionException { + DockerClient dockerClient = new DockerClient(conf); + String containerId = ctx.getContainer().getContainerId().toString(); + String commandFile = + dockerClient.writeCommandToTempFile(dockerCommand, containerId); + PrivilegedOperation dockerOp = new PrivilegedOperation( + PrivilegedOperation.OperationType.RUN_DOCKER_CMD); + dockerOp.appendArgs(commandFile); + dockerOp.disableFailureLogging(); + try { + return privilegedOperationExecutor + .executePrivilegedOperation(null, dockerOp, null, + ctx.getContainer().getLaunchContext(). + getEnvironment(), true, false).trim(); + } catch (PrivilegedOperationException e) { + throw new ContainerExecutionException("Docker operation failed", + e.getExitCode(), e.getOutput(), e.getErrorOutput()); + } + } + + /** + * Execute the docker rm command. + * + * @param ctx the container runtime context. + * @param conf the hadoop configuration. + * @param privilegedOperationExecutor the privileged operations executor. + * @throws ContainerExecutionException if the operation fails. + */ + public static void executeDockerRm(ContainerRuntimeContext ctx, + Configuration conf, + PrivilegedOperationExecutor privilegedOperationExecutor) + throws ContainerExecutionException { + String containerId = ctx.getContainer().getContainerId().toString(); + DockerRmCommand rmCommand = new DockerRmCommand(containerId); + LOG.debug("Running docker rm. ContainerId: " + containerId); + executeDockerCommand(rmCommand, ctx, conf, privilegedOperationExecutor); + } + + /** + * Execute the docker stop command. + * + * @param ctx the container runtime context. + * @param conf the hadoop configuration. + * @param privilegedOperationExecutor the privileged operations executor. + * @throws ContainerExecutionException if the operation fails. + */ + public static void executeDockerStop(ContainerRuntimeContext ctx, + Configuration conf, + PrivilegedOperationExecutor privilegedOperationExecutor) + throws ContainerExecutionException { + String containerId = ctx.getContainer().getContainerId().toString(); + DockerStopCommand stopCommand = new DockerStopCommand(containerId); + LOG.debug("Running docker stop. ContainerId: " + containerId); + executeDockerCommand(stopCommand, ctx, conf, privilegedOperationExecutor); + } + + /** + * Execute the docker inspect command. + * + * @param dockerInspectCommand the docker inspect command to execute. + * @param ctx the container runtime context. + * @param conf the hadoop configuration. + * @param privilegedOperationExecutor the privileged operations executor. + * @return the result of the docker inspect command + * @throws ContainerExecutionException if the operation fails. + */ + public static String executeDockerInspect( + DockerInspectCommand dockerInspectCommand, ContainerRuntimeContext ctx, + Configuration conf, + PrivilegedOperationExecutor privilegedOperationExecutor) + throws ContainerExecutionException { + String containerId = ctx.getContainer().getContainerId().toString(); + LOG.debug("Running docker inspect. ContainerId: " + containerId); + return executeDockerCommand(dockerInspectCommand, ctx, conf, + privilegedOperationExecutor); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerContainerStatusHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerContainerStatusHandler.java new file mode 100644 index 0000000..1fcae81 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerContainerStatusHandler.java @@ -0,0 +1,137 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker; + +import com.google.common.annotations.VisibleForTesting; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationExecutor; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerExecutionException; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerRuntimeContext; + +/** + * Encapsulate docker container status. + */ +public class DockerContainerStatusHandler { + + private static final Log LOG = + LogFactory.getLog(DockerContainerStatusHandler.class); + + private StatusState statusState; + + /** + * Potential states that the docker status can return. + */ + public enum StatusState { + RUNNING("running"), + STOPPED("stopped"), + EXITED("exited"), + NONEXISTENT("nonexistent"), + UNKNOWN("unknown"); + + private final String name; + + StatusState(String name) { + this.name = name; + } + + public String getName() { + return name; + } + } + + /** + * Set the container status. This is intended for testing only. + * + * @param statusState the status state to inject. + */ + @VisibleForTesting + public void setStatusState(StatusState statusState) { + this.statusState = statusState; + } + + /** + * Reset the container status so that previous state does not impact future + * tests. + */ + @VisibleForTesting + public void resetStatusState() { + statusState = null; + } + + /** + * Get the status of the docker container. This runs a docker inspect to + * get the status. If the container no longer exists, docker inspect throws + * an exception and the removed status is returned. If a status value has + * been injected for testing purposes, return the injected status. + * + * @param ctx the container runtime context. + * @param conf the hadoop configuration. + * @param privilegedOperationExecutor the privileged operations executor. + * @return a {@link StatusState} representing the current + * status. + */ + public StatusState getContainerStatus(ContainerRuntimeContext ctx, + Configuration conf, + PrivilegedOperationExecutor privilegedOperationExecutor) { + try { + String currentContainerStatus = + executeStatusCommand(ctx, conf, privilegedOperationExecutor); + if (currentContainerStatus.equals(StatusState.RUNNING.getName())) { + return StatusState.RUNNING; + } else if (currentContainerStatus.equals(StatusState.STOPPED.getName())) { + return StatusState.STOPPED; + } else if (currentContainerStatus.equals(StatusState.EXITED.getName())) { + return StatusState.EXITED; + } else { + return StatusState.UNKNOWN; + } + } catch (ContainerExecutionException e) { + return StatusState.NONEXISTENT; + } + } + + private String executeStatusCommand(ContainerRuntimeContext ctx, + Configuration conf, + PrivilegedOperationExecutor privilegedOperationExecutor) + throws ContainerExecutionException { + + String status; + String containerId = ctx.getContainer().getContainerId().toString(); + DockerInspectCommand dockerInspectCommand = + new DockerInspectCommand(containerId).getContainerStatus(); + try { + status = DockerCommandExecutor + .executeDockerInspect(dockerInspectCommand, ctx, conf, + privilegedOperationExecutor); + LOG.debug("Container Status: " + status + " ContainerId: " + containerId); + } catch (ContainerExecutionException e) { + throw new ContainerExecutionException( + "Container Status: " + StatusState.NONEXISTENT.getName() + + "ContainerId: " + + containerId); + } + + // Allow for injecting the container's status for testing. + if (statusState != null) { + status = statusState.getName(); + } + + return status; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c index ca3847e..b1836a4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c @@ -1286,7 +1286,6 @@ int launch_docker_container_as_user(const char * user, const char *app_id, char docker_wait_command[EXECUTOR_PATH_MAX]; char docker_logs_command[EXECUTOR_PATH_MAX]; char docker_inspect_command[EXECUTOR_PATH_MAX]; - char docker_rm_command[EXECUTOR_PATH_MAX]; int container_file_source =-1; int cred_file_source = -1; int BUFFER_SIZE = 4096; @@ -1437,19 +1436,6 @@ int launch_docker_container_as_user(const char * user, const char *app_id, } } - fprintf(LOGFILE, "Removing docker container post-exit...\n"); - snprintf(docker_rm_command, EXECUTOR_PATH_MAX, - "%s rm %s", docker_binary, container_id); - FILE* rm_docker = popen(docker_rm_command, "w"); - if (pclose (rm_docker) != 0) - { - fprintf (ERRORFILE, - "Could not remove container %s.\n", docker_rm_command); - fflush(ERRORFILE); - exit_code = UNABLE_TO_EXECUTE_CONTAINER_SCRIPT; - goto cleanup; - } - cleanup: //clean up docker command file unlink(command_file); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/MockContainerExecutorBinary.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/MockContainerExecutorBinary.java new file mode 100644 index 0000000..185b33d --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/MockContainerExecutorBinary.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux; + +import java.io.File; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.yarn.conf.YarnConfiguration; + +/** + * Configure the use of the mock container executor binary. + */ +public final class MockContainerExecutorBinary { + + private MockContainerExecutorBinary() { + } + + /** + * Return a configuration object with the mock container executor binary + * preconfigured. + * @param conf The hadoop configuration + * @return The hadoop configuration + */ + public static Configuration enableMockContainerExecutor( + Configuration conf) { + File f = new File("./src/test/resources/mock-container-executor"); + if (!FileUtil.canExecute(f)) { + FileUtil.setExecutable(f, true); + } + String executorPath = f.getAbsolutePath(); + conf.set(YarnConfiguration.NM_LINUX_CONTAINER_EXECUTOR_PATH, executorPath); + return conf; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/privileged/PrivilegedOperationCaptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/privileged/PrivilegedOperationCaptor.java new file mode 100644 index 0000000..db3b6fd --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/privileged/PrivilegedOperationCaptor.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged; + +import java.io.File; +import java.util.List; +import java.util.Map; +import org.mockito.ArgumentCaptor; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyList; +import static org.mockito.Matchers.eq; +import org.mockito.Mockito; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +/** + * Captures operations from mock {@link PrivilegedOperation} instances. + */ +public final class PrivilegedOperationCaptor { + + private PrivilegedOperationCaptor() {} + + /** + * Capture the operation that should be performed by the + * PrivilegedOperationExecutor. + * + * @param mockExecutor mock PrivilegedOperationExecutor. + * @param invocationCount number of invocations expected. + * @return a list of operations that were invoked. + * @throws PrivilegedOperationException when the operation fails to execute. + */ + @SuppressWarnings("unchecked") + public static List capturePrivilegedOperations( + PrivilegedOperationExecutor mockExecutor, int invocationCount, + boolean grabOutput) throws PrivilegedOperationException { + ArgumentCaptor opCaptor = + ArgumentCaptor.forClass(PrivilegedOperation.class); + + //one or more invocations expected + //due to type erasure + mocking, this verification requires a suppress + // warning annotation on the entire method + verify(mockExecutor, times(invocationCount)) + .executePrivilegedOperation(anyList(), opCaptor.capture(), + any(File.class), any(Map.class), eq(grabOutput), eq(false)); + + //verification completed. we need to isolate specific invications. + // hence, reset mock here + Mockito.reset(mockExecutor); + + return opCaptor.getAllValues(); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerRuntimeTestingUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerRuntimeTestingUtils.java new file mode 100644 index 0000000..5028418 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerRuntimeTestingUtils.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime; + +import java.io.IOException; +import java.nio.charset.Charset; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperation; +import org.junit.Assert; + +/** + * Utility class for commonly used docker runtime testing helpers. + */ +public final class DockerRuntimeTestingUtils { + + private DockerRuntimeTestingUtils() { + } + + /** + * Ensure that the contents of the docker command file are correct and + * return the docker commands found in the command file. + * + * @param ops list of {@link PrivilegedOperation} from the operation + * invocation. + * @return the docker commands found in the docker command file. + * @throws IOException if the docker command file cannot be read. + */ + public static List getValidatedDockerCommands( + List ops) throws IOException { + try { + List dockerCommands = new ArrayList<>(); + for (PrivilegedOperation op : ops) { + Assert.assertEquals(op.getOperationType(), + PrivilegedOperation.OperationType.RUN_DOCKER_CMD); + String dockerCommandFile = op.getArguments().get(0); + List dockerCommandFileContents = Files + .readAllLines(Paths.get(dockerCommandFile), + Charset.forName("UTF-8")); + dockerCommands.addAll(dockerCommandFileContents); + } + return dockerCommands; + } catch (IOException e) { + throw new IOException("Unable to read the docker command file.", e); + } + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java index a05ff46..90e74a5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java @@ -23,14 +23,16 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.Path; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker.DockerContainerStatusHandler; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.MockContainerExecutorBinary; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperation; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationCaptor; import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationException; import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationExecutor; import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.CGroupsHandler; @@ -68,6 +70,7 @@ private Configuration conf; private PrivilegedOperationExecutor mockExecutor; private CGroupsHandler mockCGroupsHandler; + private DockerContainerStatusHandler dockerContainerStatusHandler; private String containerId; private Container container; private ContainerId cId; @@ -171,6 +174,8 @@ public void setup() { .setExecutionAttribute(CONTAINER_LOG_DIRS, containerLogDirs) .setExecutionAttribute(LOCALIZED_RESOURCES, localizedResources) .setExecutionAttribute(RESOURCES_OPTIONS, resourcesOptions); + + dockerContainerStatusHandler = new DockerContainerStatusHandler(); } @Test @@ -798,16 +803,19 @@ public void testMountMultiple() public void testContainerLivelinessCheck() throws ContainerExecutionException, PrivilegedOperationException { - DockerLinuxContainerRuntime runtime = new DockerLinuxContainerRuntime( - mockExecutor, mockCGroupsHandler); + DockerLinuxContainerRuntime runtime = + new DockerLinuxContainerRuntime(mockExecutor, mockCGroupsHandler); builder.setExecutionAttribute(RUN_AS_USER, runAsUser) - .setExecutionAttribute(USER, user) - .setExecutionAttribute(PID, signalPid) + .setExecutionAttribute(USER, user).setExecutionAttribute(PID, signalPid) .setExecutionAttribute(SIGNAL, ContainerExecutor.Signal.NULL); - runtime.initialize(getConfigurationWithMockContainerExecutor()); + runtime.initialize( + MockContainerExecutorBinary.enableMockContainerExecutor(conf)); runtime.signalContainer(builder.build()); - PrivilegedOperation op = capturePrivilegedOperation(); + List ops = PrivilegedOperationCaptor + .capturePrivilegedOperations(mockExecutor, 1, false); + Assert.assertEquals(1, ops.size()); + PrivilegedOperation op = ops.get(0); Assert.assertEquals(op.getOperationType(), PrivilegedOperation.OperationType.SIGNAL_CONTAINER); Assert.assertEquals("run_as_user", op.getArguments().get(0)); @@ -821,62 +829,142 @@ public void testContainerLivelinessCheck() public void testDockerStopOnTermSignal() throws ContainerExecutionException, PrivilegedOperationException, IOException { - List dockerCommands = getDockerCommandsForSignal( - ContainerExecutor.Signal.TERM); - Assert.assertEquals(1, dockerCommands.size()); - Assert.assertEquals("stop container_id", dockerCommands.get(0)); + env.put( + DockerLinuxContainerRuntime.ENV_DOCKER_CONTAINER_KEEP_CONTAINER_ON_EXIT, + "true"); + List dockerCommands = + getDockerCommandsForSignal(ContainerExecutor.Signal.TERM, 2, + DockerContainerStatusHandler.StatusState.RUNNING); + Assert.assertEquals(2, dockerCommands.size()); + Assert.assertEquals("inspect --format='{{.State.Status}}' container_id", + dockerCommands.get(0)); + Assert.assertEquals("stop container_id", dockerCommands.get(1)); } @Test public void testDockerStopOnKillSignal() throws ContainerExecutionException, PrivilegedOperationException, IOException { - List dockerCommands = getDockerCommandsForSignal( - ContainerExecutor.Signal.KILL); - Assert.assertEquals(1, dockerCommands.size()); - Assert.assertEquals("stop container_id", dockerCommands.get(0)); + env.put( + DockerLinuxContainerRuntime.ENV_DOCKER_CONTAINER_KEEP_CONTAINER_ON_EXIT, + "true"); + + List dockerCommands = + getDockerCommandsForSignal(ContainerExecutor.Signal.KILL, 2, + DockerContainerStatusHandler.StatusState.RUNNING); + Assert.assertEquals(2, dockerCommands.size()); + Assert.assertEquals("inspect --format='{{.State.Status}}' container_id", + dockerCommands.get(0)); + Assert.assertEquals("stop container_id", dockerCommands.get(1)); } @Test public void testDockerStopOnQuitSignal() throws ContainerExecutionException, PrivilegedOperationException, IOException { - List dockerCommands = getDockerCommandsForSignal( - ContainerExecutor.Signal.QUIT); - Assert.assertEquals(1, dockerCommands.size()); - Assert.assertEquals("stop container_id", dockerCommands.get(0)); + env.put( + DockerLinuxContainerRuntime.ENV_DOCKER_CONTAINER_KEEP_CONTAINER_ON_EXIT, + "true"); + List dockerCommands = + getDockerCommandsForSignal(ContainerExecutor.Signal.QUIT, 2, + DockerContainerStatusHandler.StatusState.RUNNING); + Assert.assertEquals(2, dockerCommands.size()); + Assert.assertEquals("inspect --format='{{.State.Status}}' container_id", + dockerCommands.get(0)); + Assert.assertEquals("stop container_id", dockerCommands.get(1)); } - private List getDockerCommandsForSignal( - ContainerExecutor.Signal signal) + @Test + public void testDockerStopRmOnTermSignal() throws ContainerExecutionException, PrivilegedOperationException, IOException { + List dockerCommands = + getDockerCommandsForSignal(ContainerExecutor.Signal.TERM, 3, + DockerContainerStatusHandler.StatusState.RUNNING); + Assert.assertEquals(3, dockerCommands.size()); + Assert.assertEquals("inspect --format='{{.State.Status}}' container_id", + dockerCommands.get(0)); + Assert.assertEquals("stop container_id", dockerCommands.get(1)); + Assert.assertEquals("rm container_id", dockerCommands.get(2)); + } - DockerLinuxContainerRuntime runtime = new DockerLinuxContainerRuntime( - mockExecutor, mockCGroupsHandler); - builder.setExecutionAttribute(RUN_AS_USER, runAsUser) - .setExecutionAttribute(USER, user) - .setExecutionAttribute(PID, signalPid) - .setExecutionAttribute(SIGNAL, signal); - runtime.initialize(getConfigurationWithMockContainerExecutor()); - runtime.signalContainer(builder.build()); + @Test + public void testDockerStopRmOnKillSignal() + throws ContainerExecutionException, PrivilegedOperationException, + IOException { + List dockerCommands = + getDockerCommandsForSignal(ContainerExecutor.Signal.KILL, 3, + DockerContainerStatusHandler.StatusState.RUNNING); + Assert.assertEquals(3, dockerCommands.size()); + Assert.assertEquals("inspect --format='{{.State.Status}}' container_id", + dockerCommands.get(0)); + Assert.assertEquals("stop container_id", dockerCommands.get(1)); + Assert.assertEquals("rm container_id", dockerCommands.get(2)); + } - PrivilegedOperation op = capturePrivilegedOperation(); - Assert.assertEquals(op.getOperationType(), - PrivilegedOperation.OperationType.RUN_DOCKER_CMD); - String dockerCommandFile = op.getArguments().get(0); - return Files.readAllLines(Paths.get(dockerCommandFile), - Charset.forName("UTF-8")); + @Test + public void testDockerStopRmOnQuitSignal() + throws ContainerExecutionException, PrivilegedOperationException, + IOException { + List dockerCommands = + getDockerCommandsForSignal(ContainerExecutor.Signal.QUIT, 3, + DockerContainerStatusHandler.StatusState.RUNNING); + Assert.assertEquals(3, dockerCommands.size()); + Assert.assertEquals("inspect --format='{{.State.Status}}' container_id", + dockerCommands.get(0)); + Assert.assertEquals("stop container_id", dockerCommands.get(1)); + Assert.assertEquals("rm container_id", dockerCommands.get(2)); } - private Configuration getConfigurationWithMockContainerExecutor() { - File f = new File("./src/test/resources/mock-container-executor"); - if(!FileUtil.canExecute(f)) { - FileUtil.setExecutable(f, true); - } - String executorPath = f.getAbsolutePath(); - conf.set(YarnConfiguration.NM_LINUX_CONTAINER_EXECUTOR_PATH, executorPath); - return conf; + @Test + public void testDockerNoopForNonexistentContainer() + throws ContainerExecutionException, PrivilegedOperationException, + IOException { + List dockerCommands = + getDockerCommandsForSignal(ContainerExecutor.Signal.QUIT, 1, + DockerContainerStatusHandler.StatusState.NONEXISTENT); + Assert.assertEquals(1, dockerCommands.size()); + Assert.assertEquals("inspect --format='{{.State.Status}}' container_id", + dockerCommands.get(0)); + } + + @Test + public void testDockerNoopForUnknownContainer() + throws ContainerExecutionException, PrivilegedOperationException, + IOException { + List dockerCommands = + getDockerCommandsForSignal(ContainerExecutor.Signal.QUIT, 1, + DockerContainerStatusHandler.StatusState.UNKNOWN); + Assert.assertEquals(1, dockerCommands.size()); + Assert.assertEquals("inspect --format='{{.State.Status}}' container_id", + dockerCommands.get(0)); + } + + private List getDockerCommandsForSignal( + ContainerExecutor.Signal signal, int expectedCommandCount, + DockerContainerStatusHandler.StatusState status) + throws ContainerExecutionException, PrivilegedOperationException, + IOException { + DockerLinuxContainerRuntime runtime = getRuntimeForSignal(signal, status); + runtime.signalContainer(builder.build()); + List ops = PrivilegedOperationCaptor + .capturePrivilegedOperations(mockExecutor, expectedCommandCount, true); + return DockerRuntimeTestingUtils.getValidatedDockerCommands(ops); } + private DockerLinuxContainerRuntime getRuntimeForSignal( + ContainerExecutor.Signal signal, + DockerContainerStatusHandler.StatusState status) + throws ContainerExecutionException { + dockerContainerStatusHandler.setStatusState(status); + DockerLinuxContainerRuntime runtime = + new DockerLinuxContainerRuntime(mockExecutor, mockCGroupsHandler, + dockerContainerStatusHandler); + builder.setExecutionAttribute(RUN_AS_USER, runAsUser) + .setExecutionAttribute(USER, user).setExecutionAttribute(PID, signalPid) + .setExecutionAttribute(SIGNAL, signal); + runtime.initialize( + MockContainerExecutorBinary.enableMockContainerExecutor(conf)); + return runtime; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerCommandExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerCommandExecutor.java new file mode 100644 index 0000000..275149f --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerCommandExecutor.java @@ -0,0 +1,158 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker; + +import java.util.HashMap; +import java.util.List; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.MockContainerExecutorBinary; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperation; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationCaptor; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationExecutor; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.CGroupsHandler; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.DockerLinuxContainerRuntime; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.DockerRuntimeTestingUtils; +import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.CONTAINER_ID_STR; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerRuntimeContext; +import static org.junit.Assert.*; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mockito; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * Test common docker commands. + */ +public class TestDockerCommandExecutor { + + private static final String MOCK_CONTAINER_ID = "container_id"; + + private PrivilegedOperationExecutor mockExecutor; + private CGroupsHandler mockCGroupsHandler; + private Configuration configuration; + private ContainerRuntimeContext.Builder builder; + private DockerLinuxContainerRuntime runtime; + private Container container; + private ContainerId cId; + private ContainerLaunchContext context; + private HashMap env; + + @Before + public void setUp() throws Exception { + mockExecutor = Mockito.mock(PrivilegedOperationExecutor.class); + mockCGroupsHandler = Mockito.mock(CGroupsHandler.class); + configuration = new Configuration(); + runtime = new DockerLinuxContainerRuntime(mockExecutor, mockCGroupsHandler); + container = mock(Container.class); + cId = mock(ContainerId.class); + context = mock(ContainerLaunchContext.class); + env = new HashMap<>(); + builder = new ContainerRuntimeContext.Builder(container); + + when(container.getContainerId()).thenReturn(cId); + when(cId.toString()).thenReturn(MOCK_CONTAINER_ID); + when(container.getLaunchContext()).thenReturn(context); + when(context.getEnvironment()).thenReturn(env); + + builder.setExecutionAttribute(CONTAINER_ID_STR, MOCK_CONTAINER_ID); + runtime.initialize( + MockContainerExecutorBinary.enableMockContainerExecutor(configuration)); + } + + @Test + public void testExecuteDockerCommand() throws Exception { + DockerStopCommand dockerStopCommand = + new DockerStopCommand(MOCK_CONTAINER_ID); + DockerCommandExecutor + .executeDockerCommand(dockerStopCommand, builder.build(), configuration, + mockExecutor); + List ops = PrivilegedOperationCaptor + .capturePrivilegedOperations(mockExecutor, 1, true); + assertEquals(1, ops.size()); + assertEquals(PrivilegedOperation.OperationType.RUN_DOCKER_CMD.name(), + ops.get(0).getOperationType().name()); + } + + @Test + public void testExecuteDockerRm() throws Exception { + DockerCommandExecutor + .executeDockerRm(builder.build(), configuration, mockExecutor); + List ops = PrivilegedOperationCaptor + .capturePrivilegedOperations(mockExecutor, 1, true); + List dockerCommands = + DockerRuntimeTestingUtils.getValidatedDockerCommands(ops); + assertEquals(1, ops.size()); + assertEquals(PrivilegedOperation.OperationType.RUN_DOCKER_CMD.name(), + ops.get(0).getOperationType().name()); + assertEquals(1, dockerCommands.size()); + assertEquals("rm " + MOCK_CONTAINER_ID, dockerCommands.get(0)); + } + + @Test + public void testExecuteDockerStop() throws Exception { + DockerCommandExecutor + .executeDockerStop(builder.build(), configuration, mockExecutor); + List ops = PrivilegedOperationCaptor + .capturePrivilegedOperations(mockExecutor, 1, true); + List dockerCommands = + DockerRuntimeTestingUtils.getValidatedDockerCommands(ops); + assertEquals(1, ops.size()); + assertEquals(PrivilegedOperation.OperationType.RUN_DOCKER_CMD.name(), + ops.get(0).getOperationType().name()); + assertEquals(1, dockerCommands.size()); + assertEquals("stop " + MOCK_CONTAINER_ID, dockerCommands.get(0)); + } + + @Test + public void testExecuteDockerInspect() throws Exception { + DockerCommandExecutor + .executeDockerInspect(new DockerInspectCommand(MOCK_CONTAINER_ID), + builder.build(), configuration, mockExecutor); + List ops = PrivilegedOperationCaptor + .capturePrivilegedOperations(mockExecutor, 1, true); + List dockerCommands = + DockerRuntimeTestingUtils.getValidatedDockerCommands(ops); + assertEquals(1, ops.size()); + assertEquals(PrivilegedOperation.OperationType.RUN_DOCKER_CMD.name(), + ops.get(0).getOperationType().name()); + assertEquals(1, dockerCommands.size()); + assertEquals("inspect", dockerCommands.get(0)); + } + + @Test + public void testExecuteDockerInspectStatus() throws Exception { + DockerInspectCommand dockerInspectCommand = + new DockerInspectCommand(MOCK_CONTAINER_ID).getContainerStatus(); + DockerCommandExecutor + .executeDockerInspect(dockerInspectCommand, builder.build(), + configuration, mockExecutor); + List ops = PrivilegedOperationCaptor + .capturePrivilegedOperations(mockExecutor, 1, true); + List dockerCommands = + DockerRuntimeTestingUtils.getValidatedDockerCommands(ops); + assertEquals(1, ops.size()); + assertEquals(PrivilegedOperation.OperationType.RUN_DOCKER_CMD.name(), + ops.get(0).getOperationType().name()); + assertEquals(1, dockerCommands.size()); + assertEquals("inspect --format='{{.State.Status}}' " + MOCK_CONTAINER_ID, + dockerCommands.get(0)); + } +} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerContainerStatusHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerContainerStatusHandler.java new file mode 100644 index 0000000..5d6794e --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerContainerStatusHandler.java @@ -0,0 +1,131 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker; + +import java.util.HashMap; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.MockContainerExecutorBinary; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationExecutor; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.CGroupsHandler; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.DockerLinuxContainerRuntime; +import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.CONTAINER_ID_STR; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerRuntimeContext; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mockito; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * Test operations for the {@link DockerContainerStatusHandler}. + */ +public class TestDockerContainerStatusHandler { + + private static final String MOCK_CONTAINER_ID = "container_id"; + + private PrivilegedOperationExecutor mockExecutor; + private CGroupsHandler mockCGroupsHandler; + private DockerContainerStatusHandler dockerContainerStatusHandler; + private Configuration configuration; + private ContainerRuntimeContext.Builder builder; + private DockerLinuxContainerRuntime runtime; + private Container container; + private ContainerId cId; + private ContainerLaunchContext context; + private HashMap env; + + @Before + public void setUp() throws Exception { + mockExecutor = Mockito.mock(PrivilegedOperationExecutor.class); + mockCGroupsHandler = Mockito.mock(CGroupsHandler.class); + dockerContainerStatusHandler = new DockerContainerStatusHandler(); + configuration = new Configuration(); + runtime = new DockerLinuxContainerRuntime(mockExecutor, mockCGroupsHandler, + dockerContainerStatusHandler); + container = mock(Container.class); + cId = mock(ContainerId.class); + context = mock(ContainerLaunchContext.class); + env = new HashMap<>(); + builder = new ContainerRuntimeContext.Builder(container); + + when(container.getContainerId()).thenReturn(cId); + when(cId.toString()).thenReturn(MOCK_CONTAINER_ID); + when(container.getLaunchContext()).thenReturn(context); + when(context.getEnvironment()).thenReturn(env); + + builder.setExecutionAttribute(CONTAINER_ID_STR, MOCK_CONTAINER_ID); + runtime.initialize( + MockContainerExecutorBinary.enableMockContainerExecutor(configuration)); + } + + @Test + public void testSetDockerContainerStatusState() throws Exception { + dockerContainerStatusHandler + .setStatusState(DockerContainerStatusHandler.StatusState.RUNNING); + Assert.assertEquals(DockerContainerStatusHandler.StatusState.RUNNING, + dockerContainerStatusHandler + .getContainerStatus(builder.build(), configuration, mockExecutor)); + } + + @Test + public void testResetDockerContainerStatusState() throws Exception { + dockerContainerStatusHandler.resetStatusState(); + Assert.assertEquals(DockerContainerStatusHandler.StatusState.UNKNOWN, + dockerContainerStatusHandler + .getContainerStatus(builder.build(), configuration, mockExecutor)); + } + + @Test + public void testGetContainerStatusForRunning() throws Exception { + dockerContainerStatusHandler + .setStatusState(DockerContainerStatusHandler.StatusState.RUNNING); + Assert.assertEquals(DockerContainerStatusHandler.StatusState.RUNNING, + dockerContainerStatusHandler + .getContainerStatus(builder.build(), configuration, mockExecutor)); + } + + @Test + public void testGetContainerStatusForStopped() throws Exception { + dockerContainerStatusHandler + .setStatusState(DockerContainerStatusHandler.StatusState.STOPPED); + Assert.assertEquals(DockerContainerStatusHandler.StatusState.STOPPED, + dockerContainerStatusHandler + .getContainerStatus(builder.build(), configuration, mockExecutor)); + } + + @Test + public void testGetContainerStatusForExited() throws Exception { + dockerContainerStatusHandler + .setStatusState(DockerContainerStatusHandler.StatusState.EXITED); + Assert.assertEquals(DockerContainerStatusHandler.StatusState.EXITED, + dockerContainerStatusHandler + .getContainerStatus(builder.build(), configuration, mockExecutor)); + } + + @Test + public void testGetContainerStatusForUnknown() throws Exception { + dockerContainerStatusHandler + .setStatusState(DockerContainerStatusHandler.StatusState.UNKNOWN); + Assert.assertEquals(DockerContainerStatusHandler.StatusState.UNKNOWN, + dockerContainerStatusHandler + .getContainerStatus(builder.build(), configuration, mockExecutor)); + } +} \ No newline at end of file