commit 4891d7a66a298edcf259c228245f5388e9e03d42 Author: Eric Yang Date: Mon Mar 4 17:35:14 2019 -0500 YARN-9292. Enhance YARN Service AM to use image ID to keep image tag "latest" consistent across nodes. Contributed by Eric Yang diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java index 458a7a1..002f7c0f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java @@ -66,6 +66,7 @@ import org.apache.hadoop.yarn.service.api.records.ContainerState; import org.apache.hadoop.yarn.service.api.records.Service; import org.apache.hadoop.yarn.service.api.records.ServiceState; +import org.apache.hadoop.yarn.service.api.records.Artifact.TypeEnum; import org.apache.hadoop.yarn.service.api.records.ConfigFile; import org.apache.hadoop.yarn.service.component.ComponentRestartPolicy; import org.apache.hadoop.yarn.service.component.instance.ComponentInstance; @@ -93,6 +94,7 @@ import org.slf4j.LoggerFactory; import java.io.IOException; +import java.net.InetAddress; import java.net.InetSocketAddress; import java.net.URI; import java.net.URISyntaxException; @@ -598,10 +600,22 @@ private void createAllComponents() { long allocateId = 0; // sort components by dependencies - Collection sortedComponents = - ServiceApiUtil.sortByDependencies(app.getComponents()); - - for (org.apache.hadoop.yarn.service.api.records.Component compSpec : sortedComponents) { + Collection + sortedComponents = ServiceApiUtil + .sortByDependencies(app.getComponents()); + + for (org.apache.hadoop.yarn.service.api.records.Component compSpec : + sortedComponents) { + if (compSpec.getArtifact()!=null && compSpec.getArtifact() + .getType()==TypeEnum.DOCKER) { + String id = compSpec.getArtifact().getId(); + if (id!=null && id.endsWith("latest")) { + // Lookup image id for service to remain uniform + String imageId = getImageId(id); + LOG.info("Docker image: " + id + " maps to: " + imageId); + compSpec.getArtifact().setId(imageId); + } + } Component component = new Component(compSpec, allocateId, context); componentsById.put(allocateId, component); componentsByName.put(component.getName(), component); @@ -1141,4 +1155,46 @@ public void syncSysFs(Service yarnApp) { LOG.error("Fail to sync service spec: {}", e); } } + + private String getImageId(String name) { + Configuration conf = getConfig(); + String port = conf.get("yarn.nodemanager.webapp.address").split(":")[1]; + boolean useKerberos = UserGroupInformation.isSecurityEnabled(); + StringBuilder requestPath = new StringBuilder(); + if (YarnConfiguration.useHttps(conf)) { + requestPath.append("https://"); + } else { + requestPath.append("http://"); + } + try { + ApplicationId appId = ApplicationId.fromString(app.getId()); + ContainerId cid = ContainerId + .newContainerId(ApplicationAttemptId.newInstance(appId, 1), + System.currentTimeMillis()); + InetAddress inetAddress = InetAddress.getLocalHost(); + String node = inetAddress.getHostName(); + requestPath.append(node) + .append(":") + .append(port) + .append("/ws/v1/node/container/") + .append(cid) + .append("/docker/images/") + .append(name); + if (!useKerberos) { + requestPath.append("?user.name=") + .append(UserGroupInformation.getCurrentUser() + .getShortUserName()); + } + Builder builder = HttpUtil.connect(requestPath.toString()); + String digestId = name; + String output = builder.get(String.class); + if (output.contains("@")) { + digestId = output; + } + return digestId; + } catch (IOException | URISyntaxException | InterruptedException e) { + LOG.warn("Unable to resolve " + name + " to image id, error: {}", e); + } + return name; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java index f1da846..d0552db 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java @@ -212,10 +212,14 @@ LoggerFactory.getLogger(DockerLinuxContainerRuntime.class); // This validates that the image is a proper docker image - public static final String DOCKER_IMAGE_PATTERN = + public static final String DOCKER_IMAGE_REGEX = "^(([a-zA-Z0-9.-]+)(:\\d+)?/)?([a-z0-9_./-]+)(:[\\w.-]+)?$"; - private static final Pattern dockerImagePattern = - Pattern.compile(DOCKER_IMAGE_PATTERN); + private static final Pattern DOCKER_IMAGE_PATTERN = + Pattern.compile(DOCKER_IMAGE_REGEX); + private static final String DOCKER_IMAGE_DIGEST_REGEX = + "^(([a-zA-Z0-9.-]+)(:\\d+)?/)?([a-z0-9_./-]+)(@sha256:)([a-f0-9]{6,64})"; + private static final Pattern DOCKER_IMAGE_DIGEST_PATTERN = + Pattern.compile(DOCKER_IMAGE_DIGEST_REGEX); public static final String HOSTNAME_PATTERN = "^[a-zA-Z0-9][a-zA-Z0-9_.-]+$"; private static final Pattern hostnamePattern = Pattern.compile( @@ -1381,7 +1385,8 @@ public static void validateImageName(String imageName) throw new ContainerExecutionException( ENV_DOCKER_CONTAINER_IMAGE + " not set!"); } - if (!dockerImagePattern.matcher(imageName).matches()) { + if (!DOCKER_IMAGE_PATTERN.matcher(imageName).matches() && + !DOCKER_IMAGE_DIGEST_PATTERN.matcher(imageName).matches()) { throw new ContainerExecutionException("Image name '" + imageName + "' doesn't match docker image name pattern"); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java index 106144f..2752603 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java @@ -34,6 +34,7 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.records.AuxServiceRecords; import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.ResourcePlugin; import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.ResourcePluginManager; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerExecutionException; import org.apache.hadoop.yarn.server.nodemanager.webapp.dao.AuxiliaryServicesInfo; import org.apache.hadoop.yarn.server.nodemanager.webapp.dao.NMResourceInfo; import org.slf4j.Logger; @@ -80,6 +81,9 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationState; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationExecutor; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker.DockerCommandExecutor; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker.DockerImagesCommand; import org.apache.hadoop.yarn.server.nodemanager.webapp.dao.AppInfo; import org.apache.hadoop.yarn.server.nodemanager.webapp.dao.AppsInfo; import org.apache.hadoop.yarn.server.nodemanager.webapp.dao.ContainerInfo; @@ -706,4 +710,31 @@ private UserGroupInformation getCallerUserGroupInformation( return callerUGI; } + + @GET + @Path("/container/{id}/docker/images/{name}") + @Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, + MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 }) + public String getImageId(@PathParam("id") String id, + @PathParam("name") String name) { + DockerImagesCommand dockerImagesCommand = new DockerImagesCommand(); + dockerImagesCommand = dockerImagesCommand.getSingleImageStatus(name); + PrivilegedOperationExecutor privOpExecutor = + PrivilegedOperationExecutor.getInstance(this.nmContext.getConf()); + try { + String output = DockerCommandExecutor.executeDockerCommand( + dockerImagesCommand, id, null, privOpExecutor, false, nmContext); + String[] ids = output.substring(1, output.length()-1).split(" "); + String result = name; + for (String image : ids) { + String[] parts = image.split("@"); + if (parts[0].equals(name.substring(0, parts[0].length()))) { + result = image; + } + } + return result; + } catch (ContainerExecutionException e) { + return "latest"; + } + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c index 090d2fc..9377cb1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c @@ -669,7 +669,12 @@ free_and_exit: static int validate_docker_image_name(const char *image_name) { const char *regex_str = "^(([a-zA-Z0-9.-]+)(:[0-9]+)?/)?([a-z0-9_./-]+)(:[a-zA-Z0-9_.-]+)?$"; - return execute_regex_match(regex_str, image_name); + const char *digest_str = "^(([a-zA-Z0-9.-]+)(:\\\d+)?/)?([a-z0-9_./-]+)(@sha256:)([a-f0-9]{6,64})"; + int ret = execute_regex_match(regex_str, image_name); + if (ret != 0) { + ret = execute_regex_match(digest_str, image_name); + } + return ret; } int get_docker_pull_command(const char *command_file, const struct configuration *conf, args *args) { @@ -1756,13 +1761,16 @@ int get_docker_images_command(const char *command_file, const struct configurati goto free_and_exit; } - ret = add_to_args(args, DOCKER_IMAGES_COMMAND); - if (ret != 0) { - goto free_and_exit; - } - image_name = get_configuration_value("image", DOCKER_COMMAND_FILE_SECTION, &command_config); if (image_name) { + ret = add_to_args(args, "image"); + if (ret != 0) { + goto free_and_exit; + } + ret = add_to_args(args, "inspect"); + if (ret != 0) { + goto free_and_exit; + } if (validate_docker_image_name(image_name) != 0) { ret = INVALID_DOCKER_IMAGE_NAME; goto free_and_exit; @@ -1771,11 +1779,29 @@ int get_docker_images_command(const char *command_file, const struct configurati if (ret != 0) { goto free_and_exit; } + ret = add_to_args(args, "-f"); + if (ret != 0) { + goto free_and_exit; + } + ret = add_to_args(args, "{{.RepoDigests}}"); + if (ret != 0) { + goto free_and_exit; + } + } else { + ret = add_to_args(args, DOCKER_IMAGES_COMMAND); + if (ret != 0) { + goto free_and_exit; + } + ret = add_to_args(args, "--format={{json .}}"); + if (ret != 0) { + goto free_and_exit; + } + ret = add_to_args(args, "--filter=dangling=false"); + if (ret != 0) { + goto free_and_exit; + } } - ret = add_to_args(args, "--format={{json .}}"); - ret = add_to_args(args, "--filter=dangling=false"); - free_and_exit: free(image_name); free_configuration(&command_config); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test_docker_util.cc b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test_docker_util.cc index 3d052e4..5ccb1ab 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test_docker_util.cc +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test_docker_util.cc @@ -1971,7 +1971,7 @@ namespace ContainerExecutor { file_cmd_vec.push_back(std::make_pair( "[docker-command-execution]\n docker-command=images\n image=image-id", - "images image-id --format={{json .}} --filter=dangling=false")); + "image inspect image-id -f {{.RepoDigests}}")); std::vector > bad_file_cmd_vec; bad_file_cmd_vec.push_back(std::make_pair( diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java index dbd980b..18bfb87 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java @@ -838,4 +838,17 @@ private static String getRedirectURL(String url) { } return redirectUrl; } + + @Test + public void testGetImageId() { + WebResource r = resource(); + String expected = "latest"; + + // Access resource-1 should success + String id = r.path("ws").path("v1").path("node").path("container") + .path("container_1553111145515_0004_01_000002").path("docker") + .path("images").path("centos:latest") + .accept(MediaType.APPLICATION_JSON).get(String.class); + assertEquals(expected, id); + } } \ No newline at end of file