diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index c29707c82ee..5db0d65544e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -1925,6 +1925,10 @@ public static boolean isAclEnabled(Configuration conf) {
public static final String NM_DOCKER_IMAGE_NAME =
DOCKER_CONTAINER_RUNTIME_PREFIX + "image-name";
+ /** Default option to decide whether to pull the latest image or not. **/
+ public static final String NM_DOCKER_IMAGE_UPDATE =
+ DOCKER_CONTAINER_RUNTIME_PREFIX + "image-update";
+
/** Capabilities allowed (and added by default) for docker containers. **/
public static final String NM_DOCKER_CONTAINER_CAPABILITIES =
DOCKER_CONTAINER_RUNTIME_PREFIX + "capabilities";
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index a0e0eda74f4..20847f9856e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -1753,6 +1753,13 @@
+
+ Default option to decide whether to pull the latest image
+ or not.
+ yarn.nodemanager.runtime.linux.docker.image-update
+ false
+
+
This configuration setting determines if privileged docker
containers are allowed on this cluster. Privileged containers are granted
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
index 7fc386d9120..3e7ff8cb7bc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
@@ -33,6 +33,7 @@
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker.DockerCommandExecutor;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker.DockerExecCommand;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker.DockerKillCommand;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker.DockerPullCommand;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker.DockerRmCommand;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker.DockerStartCommand;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker.DockerVolumeCommand;
@@ -272,6 +273,7 @@
private Map csiClients = new HashMap<>();
private PrivilegedOperationExecutor privilegedOperationExecutor;
private String defaultImageName;
+ private Boolean defaultImageUpdate;
private Set allowedNetworks = new HashSet<>();
private String defaultNetwork;
private CGroupsHandler cGroupsHandler;
@@ -352,6 +354,8 @@ public void initialize(Configuration conf, Context nmContext)
defaultTmpfsMounts.clear();
defaultImageName = conf.getTrimmed(
YarnConfiguration.NM_DOCKER_IMAGE_NAME, "");
+ defaultImageUpdate = conf.getBoolean(
+ YarnConfiguration.NM_DOCKER_IMAGE_UPDATE, false);
allowedNetworks.addAll(Arrays.asList(
conf.getTrimmedStrings(
YarnConfiguration.NM_DOCKER_ALLOWED_CONTAINER_NETWORKS,
@@ -802,6 +806,7 @@ public void launchContainer(ContainerRuntimeContext ctx)
throws ContainerExecutionException {
Container container = ctx.getContainer();
ContainerId containerId = container.getContainerId();
+ String containerIdStr = containerId.toString();
Map environment = container.getLaunchContext()
.getEnvironment();
String imageName = environment.get(ENV_DOCKER_CONTAINER_IMAGE);
@@ -822,7 +827,10 @@ public void launchContainer(ContainerRuntimeContext ctx)
validateImageName(imageName);
- String containerIdStr = containerId.toString();
+ if (defaultImageUpdate) {
+ pullImageFromRemote(containerIdStr, imageName);
+ }
+
String runAsUser = ctx.getExecutionAttribute(RUN_AS_USER);
String dockerRunAsUser = runAsUser;
Path containerWorkDir = ctx.getExecutionAttribute(CONTAINER_WORK_DIR);
@@ -1379,6 +1387,29 @@ public static void validateImageName(String imageName)
}
}
+ public void pullImageFromRemote(String containerIdStr, String imageName)
+ throws ContainerExecutionException {
+ long start = System.currentTimeMillis();
+ DockerPullCommand dockerPullCommand = new DockerPullCommand(imageName);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("now pull docker image." + " image name: " + imageName + ","
+ + " container: " + containerIdStr + ".");
+ }
+
+ DockerCommandExecutor.executeDockerCommand(dockerPullCommand,
+ containerIdStr, null,
+ privilegedOperationExecutor, false, nmContext);
+
+ long end = System.currentTimeMillis();
+ long pullImageTimeMs = end - start;
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("pull docker image done with "
+ + String.valueOf(pullImageTimeMs) + "ms spent."
+ + " image name: " + imageName + ","
+ + " container: " + containerIdStr + ".");
+ }
+ }
+
private void executeLivelinessCheck(ContainerRuntimeContext ctx)
throws ContainerExecutionException {
String procFs = ctx.getExecutionAttribute(PROCFS);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md
index 2a893e4cde8..01d6bbddba4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md
@@ -131,6 +131,15 @@ The following properties should be set in yarn-site.xml:
+
+ yarn.nodemanager.runtime.linux.docker.image-update
+ false
+
+ Optional. Default option to decide whether to pull the latest image
+ or not.
+
+
+
yarn.nodemanager.runtime.linux.docker.allowed-container-networks
host,none,bridge