commit 97991c32010554988dc19376105e967ea7a3e8cc Author: Eric Yang Date: Mon Mar 4 17:35:14 2019 -0500 YARN-9292. Enhance YARN Service AM to use image ID to keep image tag "latest" consistent across nodes. Contributed by Eric Yang diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java index 458a7a1..8f1a875 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java @@ -22,6 +22,7 @@ import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; import com.sun.jersey.api.client.ClientResponse; +import com.sun.jersey.api.client.UniformInterfaceException; import com.sun.jersey.api.client.WebResource.Builder; import org.apache.commons.io.IOUtils; @@ -66,6 +67,7 @@ import org.apache.hadoop.yarn.service.api.records.ContainerState; import org.apache.hadoop.yarn.service.api.records.Service; import org.apache.hadoop.yarn.service.api.records.ServiceState; +import org.apache.hadoop.yarn.service.api.records.Artifact.TypeEnum; import org.apache.hadoop.yarn.service.api.records.ConfigFile; import org.apache.hadoop.yarn.service.component.ComponentRestartPolicy; import org.apache.hadoop.yarn.service.component.instance.ComponentInstance; @@ -93,9 +95,11 @@ import org.slf4j.LoggerFactory; import java.io.IOException; +import java.net.InetAddress; import java.net.InetSocketAddress; import java.net.URI; import java.net.URISyntaxException; +import java.net.URLEncoder; import java.nio.ByteBuffer; import java.text.MessageFormat; import java.util.Collection; @@ -598,10 +602,22 @@ private void createAllComponents() { long allocateId = 0; // sort components by dependencies - Collection sortedComponents = - ServiceApiUtil.sortByDependencies(app.getComponents()); - - for (org.apache.hadoop.yarn.service.api.records.Component compSpec : sortedComponents) { + Collection + sortedComponents = ServiceApiUtil + .sortByDependencies(app.getComponents()); + + for (org.apache.hadoop.yarn.service.api.records.Component compSpec : + sortedComponents) { + if (compSpec.getArtifact()!=null && compSpec.getArtifact() + .getType()==TypeEnum.DOCKER) { + String id = compSpec.getArtifact().getId(); + if (id!=null && id.endsWith("latest")) { + // Lookup image id for service to remain uniform + String imageId = getImageId(id); + LOG.info("Docker image: " + id + " maps to: " + imageId); + compSpec.getArtifact().setId(imageId); + } + } Component component = new Component(compSpec, allocateId, context); componentsById.put(allocateId, component); componentsByName.put(component.getName(), component); @@ -1141,4 +1157,47 @@ public void syncSysFs(Service yarnApp) { LOG.error("Fail to sync service spec: {}", e); } } + + private String getImageId(String name) { + Configuration conf = getConfig(); + String port = conf.get("yarn.nodemanager.webapp.address").split(":")[1]; + boolean useKerberos = UserGroupInformation.isSecurityEnabled(); + StringBuilder requestPath = new StringBuilder(); + if (YarnConfiguration.useHttps(conf)) { + requestPath.append("https://"); + } else { + requestPath.append("http://"); + } + try { + ApplicationId appId = ApplicationId.fromString(app.getId()); + ContainerId cid = ContainerId + .newContainerId(ApplicationAttemptId.newInstance(appId, 1), + System.currentTimeMillis()); + InetAddress inetAddress = InetAddress.getLocalHost(); + String node = inetAddress.getHostName(); + requestPath.append(node) + .append(":") + .append(port) + .append("/ws/v1/node/container/") + .append(cid) + .append("/docker/images/") + .append(URLEncoder.encode(name, "UTF-8")); + if (!useKerberos) { + requestPath.append("?user.name=") + .append(UserGroupInformation.getCurrentUser() + .getShortUserName()); + } + Builder builder = HttpUtil.connect(requestPath.toString()); + String digestId = name; + String output = builder.get(String.class); + if (output.contains("@")) { + digestId = output; + } + return digestId; + } catch (IOException | URISyntaxException | InterruptedException | + UniformInterfaceException e) { + LOG.warn("Unable to resolve " + name + " to image id, error: {}", e); + } + return name; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java index 5885290..945b16c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java @@ -201,11 +201,16 @@ LoggerFactory.getLogger(DockerLinuxContainerRuntime.class); // This validates that the image is a proper docker image - public static final String DOCKER_IMAGE_PATTERN = - "^(([a-zA-Z0-9.-]+)(:\\d+)?/)?([a-z0-9_./-]+)(:[\\w.-]+)?$"; - private static final Pattern dockerImagePattern = - Pattern.compile(DOCKER_IMAGE_PATTERN); - + public static final String DOCKER_IMAGE_PREFIX_REGEX = + "^(([a-zA-Z0-9.-]+)(:\\d+)?/)?([a-z0-9_./-]+)"; + public static final String DOCKER_IMAGE_REGEX = + DOCKER_IMAGE_PREFIX_REGEX + "(:[\\w.-]+)?$"; + private static final Pattern DOCKER_IMAGE_PATTERN = + Pattern.compile(DOCKER_IMAGE_REGEX); + private static final String DOCKER_IMAGE_DIGEST_REGEX = + DOCKER_IMAGE_PREFIX_REGEX + "(@sha256:)([a-f0-9]{6,64})"; + private static final Pattern DOCKER_IMAGE_DIGEST_PATTERN = + Pattern.compile(DOCKER_IMAGE_DIGEST_REGEX); private static final String DEFAULT_PROCFS = "/proc"; @InterfaceAudience.Private @@ -1178,7 +1183,8 @@ public static void validateImageName(String imageName) throw new ContainerExecutionException( ENV_DOCKER_CONTAINER_IMAGE + " not set!"); } - if (!dockerImagePattern.matcher(imageName).matches()) { + if (!DOCKER_IMAGE_PATTERN.matcher(imageName).matches() && + !DOCKER_IMAGE_DIGEST_PATTERN.matcher(imageName).matches()) { throw new ContainerExecutionException("Image name '" + imageName + "' doesn't match docker image name pattern"); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java index d485c55..7eb91b2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java @@ -34,6 +34,7 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.records.AuxServiceRecords; import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.ResourcePlugin; import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.ResourcePluginManager; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerExecutionException; import org.apache.hadoop.yarn.server.nodemanager.webapp.dao.AuxiliaryServicesInfo; import org.apache.hadoop.yarn.server.nodemanager.webapp.dao.NMResourceInfo; import org.slf4j.Logger; @@ -80,6 +81,9 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationState; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationExecutor; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker.DockerCommandExecutor; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker.DockerImagesCommand; import org.apache.hadoop.yarn.server.nodemanager.webapp.dao.AppInfo; import org.apache.hadoop.yarn.server.nodemanager.webapp.dao.AppsInfo; import org.apache.hadoop.yarn.server.nodemanager.webapp.dao.ContainerInfo; @@ -702,4 +706,31 @@ private UserGroupInformation getCallerUserGroupInformation( return callerUGI; } + + @GET + @Path("/container/{id}/docker/images/{name}") + @Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, + MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 }) + public String getImageId(@PathParam("id") String id, + @PathParam("name") String name) { + DockerImagesCommand dockerImagesCommand = new DockerImagesCommand(); + dockerImagesCommand = dockerImagesCommand.getSingleImageStatus(name); + PrivilegedOperationExecutor privOpExecutor = + PrivilegedOperationExecutor.getInstance(this.nmContext.getConf()); + try { + String output = DockerCommandExecutor.executeDockerCommand( + dockerImagesCommand, id, null, privOpExecutor, false, nmContext); + String[] ids = output.substring(1, output.length()-1).split(" "); + String result = name; + for (String image : ids) { + String[] parts = image.split("@"); + if (parts[0].equals(name.substring(0, parts[0].length()))) { + result = image; + } + } + return result; + } catch (ContainerExecutionException e) { + return "latest"; + } + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c index f35fd88..b6cfa41 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c @@ -605,7 +605,12 @@ free_and_exit: static int validate_docker_image_name(const char *image_name) { const char *regex_str = "^(([a-zA-Z0-9.-]+)(:[0-9]+)?/)?([a-z0-9_./-]+)(:[a-zA-Z0-9_.-]+)?$"; - return execute_regex_match(regex_str, image_name); + const char *digest_str = "^(([a-zA-Z0-9.-]+)(:\\d+)?/)?([a-z0-9_./-]+)(@sha256:)([a-f0-9]{6,64})"; + int ret = execute_regex_match(regex_str, image_name); + if (ret != 0) { + ret = execute_regex_match(digest_str, image_name); + } + return ret; } int get_docker_pull_command(const char *command_file, const struct configuration *conf, args *args) { @@ -1746,13 +1751,16 @@ int get_docker_images_command(const char *command_file, const struct configurati goto free_and_exit; } - ret = add_to_args(args, DOCKER_IMAGES_COMMAND); - if (ret != 0) { - goto free_and_exit; - } - image_name = get_configuration_value("image", DOCKER_COMMAND_FILE_SECTION, &command_config); if (image_name) { + ret = add_to_args(args, "image"); + if (ret != 0) { + goto free_and_exit; + } + ret = add_to_args(args, "inspect"); + if (ret != 0) { + goto free_and_exit; + } if (validate_docker_image_name(image_name) != 0) { ret = INVALID_DOCKER_IMAGE_NAME; goto free_and_exit; @@ -1761,11 +1769,29 @@ int get_docker_images_command(const char *command_file, const struct configurati if (ret != 0) { goto free_and_exit; } + ret = add_to_args(args, "-f"); + if (ret != 0) { + goto free_and_exit; + } + ret = add_to_args(args, "{{.RepoDigests}}"); + if (ret != 0) { + goto free_and_exit; + } + } else { + ret = add_to_args(args, DOCKER_IMAGES_COMMAND); + if (ret != 0) { + goto free_and_exit; + } + ret = add_to_args(args, "--format={{json .}}"); + if (ret != 0) { + goto free_and_exit; + } + ret = add_to_args(args, "--filter=dangling=false"); + if (ret != 0) { + goto free_and_exit; + } } - ret = add_to_args(args, "--format={{json .}}"); - ret = add_to_args(args, "--filter=dangling=false"); - free_and_exit: free(image_name); free_configuration(&command_config); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test_docker_util.cc b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test_docker_util.cc index 8e354d1..9d0a9ba 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test_docker_util.cc +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test_docker_util.cc @@ -1974,7 +1974,7 @@ namespace ContainerExecutor { file_cmd_vec.push_back(std::make_pair( "[docker-command-execution]\n docker-command=images\n image=image-id", - "images image-id --format={{json .}} --filter=dangling=false")); + "image inspect image-id -f {{.RepoDigests}}")); std::vector > bad_file_cmd_vec; bad_file_cmd_vec.push_back(std::make_pair( diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java index d2903a9..d83e6e6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java @@ -898,4 +898,17 @@ private static String getRedirectURL(String url) { } return redirectUrl; } + + @Test + public void testGetImageId() { + WebResource r = resource(); + String expected = "latest"; + + // Access resource-1 should success + String id = r.path("ws").path("v1").path("node").path("container") + .path("container_1553111145515_0004_01_000002").path("docker") + .path("images").path("centos:latest") + .accept(MediaType.APPLICATION_JSON).get(String.class); + assertEquals(expected, id); + } } \ No newline at end of file