diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java index 75a28e648b3..f42df091ce1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java @@ -321,7 +321,7 @@ public boolean useWhitelistEnv(Map env) { return false; } - private void runDockerVolumeCommand(DockerVolumeCommand dockerVolumeCommand, + private String runDockerVolumeCommand(DockerVolumeCommand dockerVolumeCommand, Container container) throws ContainerExecutionException { try { String commandFile = dockerClient.writeCommandToTempFile( @@ -335,6 +335,7 @@ private void runDockerVolumeCommand(DockerVolumeCommand dockerVolumeCommand, LOG.info("ContainerId=" + container.getContainerId() + ", docker volume output for " + dockerVolumeCommand + ": " + output); + return output; } catch (ContainerExecutionException e) { LOG.error("Error when writing command to temp file, command=" + dockerVolumeCommand, @@ -362,15 +363,73 @@ public void prepareContainer(ContainerRuntimeContext ctx) plugin.getDockerCommandPluginInstance(); if (dockerCommandPlugin != null) { DockerVolumeCommand dockerVolumeCommand = - dockerCommandPlugin.getCreateDockerVolumeCommand(ctx.getContainer()); + dockerCommandPlugin.getCreateDockerVolumeCommand( + ctx.getContainer()); if (dockerVolumeCommand != null) { runDockerVolumeCommand(dockerVolumeCommand, container); + + // After volume created, run inspect to make sure volume properly + // created. + if (dockerVolumeCommand.getSubCommand().equals( + DockerVolumeCommand.VOLUME_CREATE_SUB_COMMAND)) { + checkDockerVolumeCreated(dockerVolumeCommand, container); + } } } } } } + private void checkDockerVolumeCreated( + DockerVolumeCommand dockerVolumeCreationCommand, Container container) + throws ContainerExecutionException { + DockerVolumeCommand dockerVolumeInspectCommand = new DockerVolumeCommand( + DockerVolumeCommand.VOLUME_LS_SUB_COMMAND); + dockerVolumeInspectCommand.setFormat("{{.Name}},{{.Driver}}"); + String output = runDockerVolumeCommand(dockerVolumeInspectCommand, + container); + + // Parse output line by line and check if it matches + String volumeName = dockerVolumeCreationCommand.getVolumeName(); + String driverName = dockerVolumeCreationCommand.getDriverName(); + if (driverName == null) { + driverName = "local"; + } + + for (String line : output.split("\n")) { + line = line.trim(); + String[] arr = line.split(","); + String v = arr[0]; + String d = null; + if (arr.length > 1) { + d = arr[1]; + } + if (volumeName.equals(v) && driverName.equals(d)) { + // Good we found it. + LOG.info( + "Docker volume-name=" + volumeName + " driver-name=" + driverName + + " already exists for container=" + container + .getContainerId() + ", continue..."); + return; + } + } + + // Couldn't find the volume + String message = + " Couldn't find volume=" + volumeName + " driver=" + driverName + + " for container=" + container.getContainerId() + + ", please check error message in log to understand " + + "why this happens."; + LOG.error(message); + + if (LOG.isDebugEnabled()) { + LOG.debug("All docker volumes in the system, command=" + + dockerVolumeInspectCommand.toString()); + } + + throw new ContainerExecutionException(message); + } + private void validateContainerNetworkType(String network) throws ContainerExecutionException { if (allowedNetworks.contains(network)) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerVolumeCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerVolumeCommand.java index a477c93afce..aac76853733 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerVolumeCommand.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerVolumeCommand.java @@ -27,23 +27,50 @@ */ public class DockerVolumeCommand extends DockerCommand { public static final String VOLUME_COMMAND = "volume"; - public static final String VOLUME_CREATE_COMMAND = "create"; + public static final String VOLUME_CREATE_SUB_COMMAND = "create"; + public static final String VOLUME_LS_SUB_COMMAND = "ls"; + // Regex pattern for volume name public static final Pattern VOLUME_NAME_PATTERN = Pattern.compile( "[a-zA-Z0-9][a-zA-Z0-9_.-]*"); + private String volumeName; + private String driverName; + private String subCommand; + public DockerVolumeCommand(String subCommand) { super(VOLUME_COMMAND); + this.subCommand = subCommand; super.addCommandArguments("sub-command", subCommand); } public DockerVolumeCommand setVolumeName(String volumeName) { super.addCommandArguments("volume", volumeName); + this.volumeName = volumeName; return this; } public DockerVolumeCommand setDriverName(String driverName) { super.addCommandArguments("driver", driverName); + this.driverName = driverName; + return this; + } + + public String getVolumeName() { + return volumeName; + } + + public String getDriverName() { + return driverName; + } + + public String getSubCommand() { + return subCommand; + } + + public DockerVolumeCommand setFormat(String format) { + super.addCommandArguments("format", format); return this; } + } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/NvidiaDockerV1CommandPlugin.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/NvidiaDockerV1CommandPlugin.java index 73d70483df8..c2e315a51c1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/NvidiaDockerV1CommandPlugin.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/NvidiaDockerV1CommandPlugin.java @@ -301,7 +301,7 @@ public DockerVolumeCommand getCreateDockerVolumeCommand(Container container) if (newVolumeName != null) { DockerVolumeCommand command = new DockerVolumeCommand( - DockerVolumeCommand.VOLUME_CREATE_COMMAND); + DockerVolumeCommand.VOLUME_CREATE_SUB_COMMAND); command.setDriverName(volumeDriver); command.setVolumeName(newVolumeName); return command; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c index e88eeac35fa..a0138d19d71 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c @@ -299,29 +299,19 @@ static int value_permitted(const struct configuration* executor_cfg, int get_docker_volume_command(const char *command_file, const struct configuration *conf, char *out, const size_t outlen) { int ret = 0; - char *driver = NULL, *volume_name = NULL, *sub_command = NULL; + char *driver = NULL, *volume_name = NULL, *sub_command = NULL, *format = NULL; struct configuration command_config = {0, NULL}; ret = read_and_verify_command_file(command_file, DOCKER_VOLUME_COMMAND, &command_config); if (ret != 0) { return ret; } sub_command = get_configuration_value("sub-command", DOCKER_COMMAND_FILE_SECTION, &command_config); - if (sub_command == NULL || 0 != strcmp(sub_command, "create")) { - fprintf(ERRORFILE, "\"create\" is the only acceptable sub-command of volume.\n"); - ret = INVALID_DOCKER_VOLUME_COMMAND; - goto cleanup; - } - - volume_name = get_configuration_value("volume", DOCKER_COMMAND_FILE_SECTION, &command_config); - if (volume_name == NULL || validate_volume_name(volume_name) != 0) { - fprintf(ERRORFILE, "%s is not a valid volume name.\n", volume_name); - ret = INVALID_DOCKER_VOLUME_NAME; - goto cleanup; - } - driver = get_configuration_value("driver", DOCKER_COMMAND_FILE_SECTION, &command_config); - if (driver == NULL) { - ret = INVALID_DOCKER_VOLUME_DRIVER; + if ((sub_command == NULL) || ((0 != strcmp(sub_command, "create")) && + (0 != strcmp(sub_command, "ls")))) { + fprintf(ERRORFILE, "\"create/ls\" are the only acceptable sub-command of volume, input sub_command=\"%s\"\n", + sub_command); + ret = INVALID_DOCKER_VOLUME_COMMAND; goto cleanup; } @@ -338,42 +328,76 @@ int get_docker_volume_command(const char *command_file, const struct configurati goto cleanup; } - ret = add_to_buffer(out, outlen, " create"); - if (ret != 0) { - goto cleanup; - } + if (0 == strcmp(sub_command, "create")) { + volume_name = get_configuration_value("volume", DOCKER_COMMAND_FILE_SECTION, &command_config); + if (volume_name == NULL || validate_volume_name(volume_name) != 0) { + fprintf(ERRORFILE, "%s is not a valid volume name.\n", volume_name); + ret = INVALID_DOCKER_VOLUME_NAME; + goto cleanup; + } - ret = add_to_buffer(out, outlen, " --name="); - if (ret != 0) { - goto cleanup; - } + driver = get_configuration_value("driver", DOCKER_COMMAND_FILE_SECTION, &command_config); + if (driver == NULL) { + ret = INVALID_DOCKER_VOLUME_DRIVER; + goto cleanup; + } - ret = add_to_buffer(out, outlen, volume_name); - if (ret != 0) { - goto cleanup; - } + ret = add_to_buffer(out, outlen, " create"); + if (ret != 0) { + goto cleanup; + } - if (!value_permitted(conf, "docker.allowed.volume-drivers", driver)) { - fprintf(ERRORFILE, "%s is not permitted docker.allowed.volume-drivers\n", - driver); - ret = INVALID_DOCKER_VOLUME_DRIVER; - goto cleanup; - } + ret = add_to_buffer(out, outlen, " --name="); + if (ret != 0) { + goto cleanup; + } - ret = add_to_buffer(out, outlen, " --driver="); - if (ret != 0) { - goto cleanup; - } + ret = add_to_buffer(out, outlen, volume_name); + if (ret != 0) { + goto cleanup; + } - ret = add_to_buffer(out, outlen, driver); - if (ret != 0) { - goto cleanup; + if (!value_permitted(conf, "docker.allowed.volume-drivers", driver)) { + fprintf(ERRORFILE, "%s is not permitted docker.allowed.volume-drivers\n", + driver); + ret = INVALID_DOCKER_VOLUME_DRIVER; + goto cleanup; + } + + ret = add_to_buffer(out, outlen, " --driver="); + if (ret != 0) { + goto cleanup; + } + + ret = add_to_buffer(out, outlen, driver); + if (ret != 0) { + goto cleanup; + } + } else if (0 == strcmp(sub_command, "ls")) { + format = get_configuration_value("format", DOCKER_COMMAND_FILE_SECTION, &command_config); + + ret = add_to_buffer(out, outlen, " ls"); + if (ret != 0) { + goto cleanup; + } + + if (format) { + ret = add_to_buffer(out, outlen, " --format="); + if (ret != 0) { + goto cleanup; + } + ret = add_to_buffer(out, outlen, format); + if (ret != 0) { + goto cleanup; + } + } } cleanup: free(driver); free(volume_name); free(sub_command); + free(format); // clean up out buffer if (ret != 0) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test_docker_util.cc b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test_docker_util.cc index 96b5d40a7b7..0c1c4bff69c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test_docker_util.cc +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test_docker_util.cc @@ -1132,12 +1132,15 @@ namespace ContainerExecutor { file_cmd_vec.push_back(std::make_pair( "[docker-command-execution]\n docker-command=volume\n sub-command=create\n volume=volume1 \n driver=driver1", "volume create --name=volume1 --driver=driver1")); + file_cmd_vec.push_back(std::make_pair( + "[docker-command-execution]\n docker-command=volume\n format={{.Name}},{{.Driver}}\n sub-command=ls", + "volume ls --format={{.Name}},{{.Driver}}")); std::vector > bad_file_cmd_vec; // Wrong subcommand bad_file_cmd_vec.push_back(std::make_pair( - "[docker-command-execution]\n docker-command=volume\n sub-command=ls\n volume=volume1 \n driver=driver1", + "[docker-command-execution]\n docker-command=volume\n sub-command=inspect\n volume=volume1 \n driver=driver1", static_cast(INVALID_DOCKER_VOLUME_COMMAND))); // Volume not specified diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java index 76aca04570b..389e8057824 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java @@ -1192,7 +1192,7 @@ private void checkVolumeCreateCommand() //single invocation expected //due to type erasure + mocking, this verification requires a suppress // warning annotation on the entire method - verify(mockExecutor, times(1)) + verify(mockExecutor, times(2)) .executePrivilegedOperation(anyList(), opCaptor.capture(), any( File.class), anyMap(), anyBoolean(), anyBoolean()); @@ -1200,7 +1200,9 @@ private void checkVolumeCreateCommand() // hence, reset mock here Mockito.reset(mockExecutor); - PrivilegedOperation op = opCaptor.getValue(); + List allCaptuures = opCaptor.getAllValues(); + + PrivilegedOperation op = allCaptuures.get(0); Assert.assertEquals(PrivilegedOperation.OperationType .RUN_DOCKER_CMD, op.getOperationType()); @@ -1210,12 +1212,33 @@ private void checkVolumeCreateCommand() Assert.assertEquals("[docker-command-execution]\n" + " docker-command=volume\n" + " sub-command=create\n" + " volume=volume1\n", fileContent); + fileInputStream.close(); + + op = allCaptuures.get(1); + Assert.assertEquals(PrivilegedOperation.OperationType + .RUN_DOCKER_CMD, op.getOperationType()); + + commandFile = new File(StringUtils.join(",", op.getArguments())); + fileInputStream = new FileInputStream(commandFile); + fileContent = new String(IOUtils.toByteArray(fileInputStream)); + Assert.assertEquals("[docker-command-execution]\n" + + " docker-command=volume\n" + " format={{.Name}},{{.Driver}}\n" + + " sub-command=ls\n", fileContent); + fileInputStream.close(); } @Test public void testDockerCommandPlugin() throws Exception { DockerLinuxContainerRuntime runtime = new DockerLinuxContainerRuntime(mockExecutor, mockCGroupsHandler); + when(mockExecutor + .executePrivilegedOperation(anyList(), any(PrivilegedOperation.class), + any(File.class), anyMap(), anyBoolean(), anyBoolean())).thenReturn( + null); + when(mockExecutor + .executePrivilegedOperation(anyList(), any(PrivilegedOperation.class), + any(File.class), anyMap(), anyBoolean(), anyBoolean())).thenReturn( + "volume1,local"); Context nmContext = mock(Context.class); ResourcePluginManager rpm = mock(ResourcePluginManager.class);