diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml index 1445d2a..6a6a1bf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml @@ -168,6 +168,16 @@ org.fusesource.leveldbjni leveldbjni-all + + org.apache.hadoop + hadoop-hdfs + provided + + + org.apache.hadoop + hadoop-hdfs-client + provided + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java index b70a4e1..9b5da07 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java @@ -27,6 +27,8 @@ import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.authorize.AccessControlList; import org.apache.hadoop.util.StringUtils; @@ -481,6 +483,12 @@ public void launchContainer(ContainerRuntimeContext ctx) } } + if (this.conf.getBoolean(HdfsClientConfigKeys.Read.ShortCircuit.KEY, + HdfsClientConfigKeys.Read.ShortCircuit.DEFAULT)) { + String socketPath = conf.get(DFSConfigKeys.DFS_DOMAIN_SOCKET_PATH_KEY, DFSConfigKeys.DFS_DOMAIN_SOCKET_PATH_DEFAULT); + runCommand.addMountLocation(socketPath, socketPath, true); + } + if (allowPrivilegedContainerExecution(container)) { runCommand.setPrivileged(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java index 3253394..e4d935c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java @@ -25,6 +25,8 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.conf.YarnConfiguration; @@ -859,6 +861,54 @@ public void testDockerStopOnQuitSignal() Assert.assertEquals("stop container_id", dockerCommands.get(0)); } + @Test + public void testDockerContainerLaunchWithShortCircuit() + throws ContainerExecutionException, PrivilegedOperationException, + IOException { + DockerLinuxContainerRuntime runtime = new DockerLinuxContainerRuntime( + mockExecutor, mockCGroupsHandler); + conf.setBoolean(HdfsClientConfigKeys.Read.ShortCircuit.KEY, true); + conf.set(DFSConfigKeys.DFS_DOMAIN_SOCKET_PATH_KEY, "/var/lib/hadoop-hdfs/dn_socket"); + runtime.initialize(conf); + + runtime.launchContainer(builder.build()); + + PrivilegedOperation op = capturePrivilegedOperationAndVerifyArgs(); + List args = op.getArguments(); + String dockerCommandFile = args.get(11); + + //This is the expected docker invocation for this case + StringBuffer expectedCommandTemplate = new StringBuffer("run --name=%1$s ") + .append("--user=%2$s -d ") + .append("--workdir=%3$s ") + .append("--net=host ") + .append(getExpectedTestCapabilitiesArgumentString()) + .append("-v %4$s:%4$s ") + .append("-v %5$s:%5$s ") + .append("-v %6$s:%6$s ") + .append("-v %7$s:%7$s ") + .append("-v %8$s:%8$s ") + .append("-v %9$s:%9$s ") + .append("%10$s ") + .append("bash %11$s/launch_container.sh"); + + String expectedCommand = String + .format(expectedCommandTemplate.toString(), containerId, runAsUser, + containerWorkDir, containerLocalDirs.get(0), filecacheDirs.get(0), + containerWorkDir, containerLogDirs.get(0), userLocalDirs.get(0), + "/var/lib/hadoop-hdfs/dn_socket", + image, containerWorkDir); + + List dockerCommands = Files.readAllLines(Paths.get + (dockerCommandFile), Charset.forName("UTF-8")); + + Assert.assertEquals(1, dockerCommands.size()); + Assert.assertEquals(expectedCommand, dockerCommands.get(0)); + + conf.setBoolean(HdfsClientConfigKeys.Read.ShortCircuit.KEY, HdfsClientConfigKeys.Read.ShortCircuit.DEFAULT); + conf.set(DFSConfigKeys.DFS_DOMAIN_SOCKET_PATH_KEY, DFSConfigKeys.DFS_DOMAIN_SOCKET_PATH_DEFAULT); + } + private List getDockerCommandsForSignal( ContainerExecutor.Signal signal) throws ContainerExecutionException, PrivilegedOperationException,