diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java index 75a28e648b3..068993207e5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java @@ -321,7 +321,7 @@ public boolean useWhitelistEnv(Map env) { return false; } - private void runDockerVolumeCommand(DockerVolumeCommand dockerVolumeCommand, + private String runDockerVolumeCommand(DockerVolumeCommand dockerVolumeCommand, Container container) throws ContainerExecutionException { try { String commandFile = dockerClient.writeCommandToTempFile( @@ -335,6 +335,7 @@ private void runDockerVolumeCommand(DockerVolumeCommand dockerVolumeCommand, LOG.info("ContainerId=" + container.getContainerId() + ", docker volume output for " + dockerVolumeCommand + ": " + output); + return output; } catch (ContainerExecutionException e) { LOG.error("Error when writing command to temp file, command=" + dockerVolumeCommand, @@ -362,15 +363,72 @@ public void prepareContainer(ContainerRuntimeContext ctx) plugin.getDockerCommandPluginInstance(); if (dockerCommandPlugin != null) { DockerVolumeCommand dockerVolumeCommand = - dockerCommandPlugin.getCreateDockerVolumeCommand(ctx.getContainer()); + dockerCommandPlugin.getCreateDockerVolumeCommand( + ctx.getContainer()); if (dockerVolumeCommand != null) { runDockerVolumeCommand(dockerVolumeCommand, container); + + // After volume created, run inspect to make sure volume properly + // created. + if (dockerVolumeCommand.getSubCommand().equals( + DockerVolumeCommand.VOLUME_CREATE_SUB_COMMAND)) { + checkDockerVolumeCreated(dockerVolumeCommand, container); + } } } } } } + private void checkDockerVolumeCreated( + DockerVolumeCommand dockerVolumeCreationCommand, Container container) + throws ContainerExecutionException { + DockerVolumeCommand dockerVolumeInspectCommand = new DockerVolumeCommand( + DockerVolumeCommand.VOLUME_INSPECT_SUB_COMMAND); + dockerVolumeInspectCommand.setFormat("{{.Name}}\t{{.Driver}}"); + String output = runDockerVolumeCommand(dockerVolumeInspectCommand, + container); + + // Parse output line by line and check if it matches + String volumeName = dockerVolumeCreationCommand.getVolumeName(); + String driverName = dockerVolumeCreationCommand.getDriverName(); + if (driverName == null) { + driverName = "local"; + } + + for (String line : output.split("\n")) { + line = line.trim(); + String[] arr = line.split("\t"); + String v = arr[0]; + String d = null; + if (arr.length > 1) { + d = arr[1]; + } + if (volumeName.equals(v) && driverName.equals(d)) { + // Good we found it. + LOG.info( + "Docker volume-name=" + volumeName + " driver-name=" + driverName + + " already exists for container=" + container + .getContainerId() + ", continue..."); + } + } + + // Couldn't find the volume + String message = + " Couldn't find volume=" + volumeName + " driver=" + driverName + + " for container=" + container.getContainerId() + + ", please check error message in log to understand " + + "why this happens."; + LOG.error(message); + + if (LOG.isDebugEnabled()) { + LOG.debug("All docker volumes in the system, command=" + + dockerVolumeInspectCommand.toString()); + } + + throw new ContainerExecutionException(message); + } + private void validateContainerNetworkType(String network) throws ContainerExecutionException { if (allowedNetworks.contains(network)) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerVolumeCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerVolumeCommand.java index a477c93afce..c1e928c1ab8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerVolumeCommand.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerVolumeCommand.java @@ -27,23 +27,50 @@ */ public class DockerVolumeCommand extends DockerCommand { public static final String VOLUME_COMMAND = "volume"; - public static final String VOLUME_CREATE_COMMAND = "create"; + public static final String VOLUME_CREATE_SUB_COMMAND = "create"; + public static final String VOLUME_INSPECT_SUB_COMMAND = "inspect"; + // Regex pattern for volume name public static final Pattern VOLUME_NAME_PATTERN = Pattern.compile( "[a-zA-Z0-9][a-zA-Z0-9_.-]*"); + private String volumeName; + private String driverName; + private String subCommand; + public DockerVolumeCommand(String subCommand) { super(VOLUME_COMMAND); + this.subCommand = subCommand; super.addCommandArguments("sub-command", subCommand); } public DockerVolumeCommand setVolumeName(String volumeName) { super.addCommandArguments("volume", volumeName); + this.volumeName = volumeName; return this; } public DockerVolumeCommand setDriverName(String driverName) { super.addCommandArguments("driver", driverName); + this.driverName = driverName; + return this; + } + + public String getVolumeName() { + return volumeName; + } + + public String getDriverName() { + return driverName; + } + + public String getSubCommand() { + return subCommand; + } + + public DockerVolumeCommand setFormat(String format) { + super.addCommandArguments("format", format); return this; } + } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/NvidiaDockerV1CommandPlugin.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/NvidiaDockerV1CommandPlugin.java index 73d70483df8..c2e315a51c1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/NvidiaDockerV1CommandPlugin.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/NvidiaDockerV1CommandPlugin.java @@ -301,7 +301,7 @@ public DockerVolumeCommand getCreateDockerVolumeCommand(Container container) if (newVolumeName != null) { DockerVolumeCommand command = new DockerVolumeCommand( - DockerVolumeCommand.VOLUME_CREATE_COMMAND); + DockerVolumeCommand.VOLUME_CREATE_SUB_COMMAND); command.setDriverName(volumeDriver); command.setVolumeName(newVolumeName); return command; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c index e88eeac35fa..4619e766e10 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c @@ -299,15 +299,16 @@ static int value_permitted(const struct configuration* executor_cfg, int get_docker_volume_command(const char *command_file, const struct configuration *conf, char *out, const size_t outlen) { int ret = 0; - char *driver = NULL, *volume_name = NULL, *sub_command = NULL; + char *driver = NULL, *volume_name = NULL, *sub_command = NULL, *format = NULL; struct configuration command_config = {0, NULL}; ret = read_and_verify_command_file(command_file, DOCKER_VOLUME_COMMAND, &command_config); if (ret != 0) { return ret; } sub_command = get_configuration_value("sub-command", DOCKER_COMMAND_FILE_SECTION, &command_config); - if (sub_command == NULL || 0 != strcmp(sub_command, "create")) { - fprintf(ERRORFILE, "\"create\" is the only acceptable sub-command of volume.\n"); + if (sub_command == NULL || 0 != strcmp(sub_command, "create") || + 0 != strcmp(sub_command, "inspect")) { + fprintf(ERRORFILE, "\"create/inspect\" are the only acceptable sub-command of volume.\n"); ret = INVALID_DOCKER_VOLUME_COMMAND; goto cleanup; } @@ -325,6 +326,8 @@ int get_docker_volume_command(const char *command_file, const struct configurati goto cleanup; } + format = get_configuration_value("format", DOCKER_COMMAND_FILE_SECTION, &command_config); + memset(out, 0, outlen); ret = add_docker_config_param(&command_config, out, outlen); @@ -338,42 +341,56 @@ int get_docker_volume_command(const char *command_file, const struct configurati goto cleanup; } - ret = add_to_buffer(out, outlen, " create"); - if (ret != 0) { - goto cleanup; - } + if (0 == strcmp(sub_command, "create")) { + ret = add_to_buffer(out, outlen, " create"); + if (ret != 0) { + goto cleanup; + } - ret = add_to_buffer(out, outlen, " --name="); - if (ret != 0) { - goto cleanup; - } + ret = add_to_buffer(out, outlen, " --name="); + if (ret != 0) { + goto cleanup; + } - ret = add_to_buffer(out, outlen, volume_name); - if (ret != 0) { - goto cleanup; - } + ret = add_to_buffer(out, outlen, volume_name); + if (ret != 0) { + goto cleanup; + } - if (!value_permitted(conf, "docker.allowed.volume-drivers", driver)) { - fprintf(ERRORFILE, "%s is not permitted docker.allowed.volume-drivers\n", - driver); - ret = INVALID_DOCKER_VOLUME_DRIVER; - goto cleanup; - } + if (!value_permitted(conf, "docker.allowed.volume-drivers", driver)) { + fprintf(ERRORFILE, "%s is not permitted docker.allowed.volume-drivers\n", + driver); + ret = INVALID_DOCKER_VOLUME_DRIVER; + goto cleanup; + } - ret = add_to_buffer(out, outlen, " --driver="); - if (ret != 0) { - goto cleanup; - } + ret = add_to_buffer(out, outlen, " --driver="); + if (ret != 0) { + goto cleanup; + } - ret = add_to_buffer(out, outlen, driver); - if (ret != 0) { - goto cleanup; + ret = add_to_buffer(out, outlen, driver); + if (ret != 0) { + goto cleanup; + } + } else if (0 == strcmp(sub_command, "inspect")) { + if (format) { + ret = add_to_buffer(out, outlen, " --format="); + if (ret != 0) { + goto cleanup; + } + ret = add_to_buffer(out, outlen, format); + if (ret != 0) { + goto cleanup; + } + } } cleanup: free(driver); free(volume_name); free(sub_command); + free(format); // clean up out buffer if (ret != 0) {