diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index cefbbf9..be4c023 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -1180,6 +1180,22 @@ private static void addDeprecatedKeys() { /** Prefix for runtime configuration constants. */ public static final String LINUX_CONTAINER_RUNTIME_PREFIX = NM_PREFIX + "runtime.linux."; + + /** + * Comma separated list of runtimes that are allowed when using + * LinuxContainerExecutor. The allowed values are: + * + */ + public static final String LINUX_CONTAINER_RUNTIME_ALLOWED_RUNTIMES = + LINUX_CONTAINER_RUNTIME_PREFIX + "allowed-runtimes"; + + /** The default list of allowed runtimes when using LinuxContainerExecutor. */ + public static final String[] DEFAULT_LINUX_CONTAINER_RUNTIME_ALLOWED_RUNTIMES + = {"default"}; + public static final String DOCKER_CONTAINER_RUNTIME_PREFIX = LINUX_CONTAINER_RUNTIME_PREFIX + "docker."; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index a399f03..881ca8e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -1461,6 +1461,14 @@ + Comma separated list of runtimes that are allowed when using + LinuxContainerExecutor. The allowed values are default and docker. + + yarn.nodemanager.runtime.linux.allowed-runtimes + default + + + This configuration setting determines the capabilities assigned to docker containers when they are launched. While these may not be case-sensitive from a docker perspective, it is best to keep these diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java index 7e51bf3..123ee44 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java @@ -46,6 +46,7 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerDiagnosticsUpdateEvent; import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerExecutionException; import org.apache.hadoop.yarn.server.nodemanager.util.NodeManagerHardwareUtils; import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerLivenessContext; import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerReacquisitionContext; @@ -252,7 +253,7 @@ public int reacquireContainer(ContainerReacquisitionContext ctx) */ public void writeLaunchEnv(OutputStream out, Map environment, Map> resources, List command, Path logDir) - throws IOException { + throws IOException, ContainerExecutionException { this.writeLaunchEnv(out, environment, resources, command, logDir, ContainerLaunch.CONTAINER_SCRIPT); } @@ -261,7 +262,7 @@ public void writeLaunchEnv(OutputStream out, Map environment, public void writeLaunchEnv(OutputStream out, Map environment, Map> resources, List command, Path logDir, String outFilename) - throws IOException { + throws IOException, ContainerExecutionException { updateEnvForWhitelistVars(environment); ContainerLaunch.ShellScriptBuilder sb = @@ -451,7 +452,8 @@ protected boolean isContainerActive(ContainerId containerId) { * @param env the environment to update * @see org.apache.hadoop.yarn.conf.YarnConfiguration#NM_ENV_WHITELIST */ - protected void updateEnvForWhitelistVars(Map env) { + protected void updateEnvForWhitelistVars(Map env) + throws ContainerExecutionException { for(String var : whitelistVars) { if (!env.containsKey(var)) { String val = getNMEnvVar(var); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java index e39592b..9cefebd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java @@ -286,7 +286,8 @@ public void buildMainArgs(List command, String user, String appId, } @Override - protected void updateEnvForWhitelistVars(Map env) { + protected void updateEnvForWhitelistVars(Map env) + throws ContainerExecutionException { if (linuxContainerRuntime.useWhitelistEnv(env)) { super.updateEnvForWhitelistVars(env); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DelegatingLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DelegatingLinuxContainerRuntime.java index e08eb6e..cf700da 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DelegatingLinuxContainerRuntime.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DelegatingLinuxContainerRuntime.java @@ -20,16 +20,19 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime; +import com.google.common.annotations.VisibleForTesting; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationExecutor; import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerExecutionException; import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerRuntimeContext; +import java.util.EnumSet; import java.util.Map; @InterfaceAudience.Private @@ -39,42 +42,66 @@ .getLog(DelegatingLinuxContainerRuntime.class); private DefaultLinuxContainerRuntime defaultLinuxContainerRuntime; private DockerLinuxContainerRuntime dockerLinuxContainerRuntime; + private EnumSet allowedRuntimes = + EnumSet.noneOf(LinuxContainerRuntimeConstants.RuntimeType.class); @Override public void initialize(Configuration conf) throws ContainerExecutionException { - PrivilegedOperationExecutor privilegedOperationExecutor = - PrivilegedOperationExecutor.getInstance(conf); - - defaultLinuxContainerRuntime = new DefaultLinuxContainerRuntime( - privilegedOperationExecutor); - defaultLinuxContainerRuntime.initialize(conf); - dockerLinuxContainerRuntime = new DockerLinuxContainerRuntime( - privilegedOperationExecutor); - dockerLinuxContainerRuntime.initialize(conf); + String[] configuredRuntimes = conf.getTrimmedStrings( + YarnConfiguration.LINUX_CONTAINER_RUNTIME_ALLOWED_RUNTIMES, + YarnConfiguration.DEFAULT_LINUX_CONTAINER_RUNTIME_ALLOWED_RUNTIMES); + for (String configuredRuntime : configuredRuntimes) { + try { + allowedRuntimes.add( + LinuxContainerRuntimeConstants.RuntimeType.valueOf( + configuredRuntime.toUpperCase())); + } catch (IllegalArgumentException e) { + throw new ContainerExecutionException("Invalid runtime set in " + + YarnConfiguration.LINUX_CONTAINER_RUNTIME_ALLOWED_RUNTIMES + " : " + + configuredRuntime); + } + } + if (isRuntimeAllowed(LinuxContainerRuntimeConstants.RuntimeType.DOCKER)) { + dockerLinuxContainerRuntime = new DockerLinuxContainerRuntime( + PrivilegedOperationExecutor.getInstance(conf)); + dockerLinuxContainerRuntime.initialize(conf); + } + if (isRuntimeAllowed(LinuxContainerRuntimeConstants.RuntimeType.DEFAULT)) { + defaultLinuxContainerRuntime = new DefaultLinuxContainerRuntime( + PrivilegedOperationExecutor.getInstance(conf)); + defaultLinuxContainerRuntime.initialize(conf); + } } @Override - public boolean useWhitelistEnv(Map env) { + public boolean useWhitelistEnv(Map env) + throws ContainerExecutionException { LinuxContainerRuntime runtime = pickContainerRuntime(env); return runtime.useWhitelistEnv(env); } - private LinuxContainerRuntime pickContainerRuntime(Container container) { + private LinuxContainerRuntime pickContainerRuntime(Container container) + throws ContainerExecutionException { return pickContainerRuntime(container.getLaunchContext().getEnvironment()); } - private LinuxContainerRuntime pickContainerRuntime(Map env) { + @VisibleForTesting + LinuxContainerRuntime pickContainerRuntime( + Map environment) throws ContainerExecutionException { LinuxContainerRuntime runtime; - - if (DockerLinuxContainerRuntime.isDockerContainerRequested(env)){ + if (dockerLinuxContainerRuntime != null && + DockerLinuxContainerRuntime.isDockerContainerRequested(environment)){ runtime = dockerLinuxContainerRuntime; - } else { + } else if (defaultLinuxContainerRuntime != null && + !DockerLinuxContainerRuntime.isDockerContainerRequested(environment)) { runtime = defaultLinuxContainerRuntime; + } else { + throw new ContainerExecutionException("Requested runtime not allowed."); } - if (LOG.isInfoEnabled()) { - LOG.info("Using container runtime: " + runtime.getClass() + if (LOG.isDebugEnabled()) { + LOG.debug("Using container runtime: " + runtime.getClass() .getSimpleName()); } @@ -116,4 +143,10 @@ public void reapContainer(ContainerRuntimeContext ctx) runtime.reapContainer(ctx); } + + @VisibleForTesting + boolean isRuntimeAllowed( + LinuxContainerRuntimeConstants.RuntimeType runtimeType) { + return allowedRuntimes.contains(runtimeType); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/LinuxContainerRuntimeConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/LinuxContainerRuntimeConstants.java index d2069a9..87f1d11 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/LinuxContainerRuntimeConstants.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/LinuxContainerRuntimeConstants.java @@ -31,6 +31,14 @@ private LinuxContainerRuntimeConstants() { } + /** + * Linux container runtime types for {@link DelegatingLinuxContainerRuntime}. + */ + public enum RuntimeType { + DEFAULT, + DOCKER + } + public static final Attribute LOCALIZED_RESOURCES = Attribute .attribute(Map.class, "localized_resources"); public static final Attribute CONTAINER_LAUNCH_PREFIX_COMMANDS = diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/runtime/ContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/runtime/ContainerRuntime.java index d54c205..0ac5833 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/runtime/ContainerRuntime.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/runtime/ContainerRuntime.java @@ -56,6 +56,8 @@ void reapContainer(ContainerRuntimeContext ctx) * @param env the container's environment variables * @return true if whitelist variables should be propagated, false otherwise * @see org.apache.hadoop.yarn.conf.YarnConfiguration#NM_ENV_WHITELIST + * @throws ContainerExecutionException if the runtime cannot be determined. */ - boolean useWhitelistEnv(Map env); + boolean useWhitelistEnv(Map env) + throws ContainerExecutionException; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java index 6bf7e3e..c028dfa 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java @@ -139,7 +139,7 @@ public void setup() throws IOException { } @Test - public void testSpecialCharSymlinks() throws IOException { + public void testSpecialCharSymlinks() throws Exception { File shellFile = null; File tempFile = null; @@ -211,7 +211,7 @@ public void testSpecialCharSymlinks() throws IOException { // test the diagnostics are generated @Test (timeout = 20000) - public void testInvalidSymlinkDiagnostics() throws IOException { + public void testInvalidSymlinkDiagnostics() throws Exception { File shellFile = null; File tempFile = null; @@ -377,7 +377,7 @@ protected String getNMEnvVar(String varname) { } @Test (timeout = 20000) - public void testInvalidEnvSyntaxDiagnostics() throws IOException { + public void testInvalidEnvSyntaxDiagnostics() throws Exception { File shellFile = null; try { @@ -454,7 +454,7 @@ public void testEnvExpansion() throws IOException { } @Test (timeout = 20000) - public void testContainerLaunchStdoutAndStderrDiagnostics() throws IOException { + public void testContainerLaunchStdoutAndStderrDiagnostics() throws Exception { File shellFile = null; try { @@ -1335,7 +1335,7 @@ public void testKillProcessGroup() throws Exception { } @Test - public void testDebuggingInformation() throws IOException { + public void testDebuggingInformation() throws Exception { File shellFile = null; File tempFile = null; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDelegatingLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDelegatingLinuxContainerRuntime.java new file mode 100644 index 0000000..f1efe66 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDelegatingLinuxContainerRuntime.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.junit.Before; +import org.junit.Test; + +import java.util.HashMap; +import java.util.Map; + +import static org.junit.Assert.*; + +/** + * Test container runtime delegation. + */ +public class TestDelegatingLinuxContainerRuntime { + + private DelegatingLinuxContainerRuntime delegatingLinuxContainerRuntime; + private Configuration conf; + + @Before + public void setUp() throws Exception { + delegatingLinuxContainerRuntime = new DelegatingLinuxContainerRuntime(); + conf = new Configuration(); + } + + @Test + public void testIsRuntimeAllowedDefault() throws Exception { + conf.set(YarnConfiguration.LINUX_CONTAINER_RUNTIME_ALLOWED_RUNTIMES, + YarnConfiguration.DEFAULT_LINUX_CONTAINER_RUNTIME_ALLOWED_RUNTIMES[0]); + delegatingLinuxContainerRuntime.initialize(conf); + assertTrue(delegatingLinuxContainerRuntime.isRuntimeAllowed( + LinuxContainerRuntimeConstants.RuntimeType.DEFAULT)); + assertFalse(delegatingLinuxContainerRuntime.isRuntimeAllowed( + LinuxContainerRuntimeConstants.RuntimeType.DOCKER)); + } + + @Test + public void testIsRuntimeAllowedDocker() throws Exception { + conf.set(YarnConfiguration.LINUX_CONTAINER_RUNTIME_ALLOWED_RUNTIMES, + "docker"); + delegatingLinuxContainerRuntime.initialize(conf); + assertTrue(delegatingLinuxContainerRuntime.isRuntimeAllowed( + LinuxContainerRuntimeConstants.RuntimeType.DOCKER)); + assertFalse(delegatingLinuxContainerRuntime.isRuntimeAllowed( + LinuxContainerRuntimeConstants.RuntimeType.DEFAULT)); + } + + + @Test + public void testIsRuntimeAllowedMultiple() throws Exception { + conf.set(YarnConfiguration.LINUX_CONTAINER_RUNTIME_ALLOWED_RUNTIMES, + "default,docker"); + delegatingLinuxContainerRuntime.initialize(conf); + assertTrue(delegatingLinuxContainerRuntime.isRuntimeAllowed( + LinuxContainerRuntimeConstants.RuntimeType.DOCKER)); + assertTrue(delegatingLinuxContainerRuntime.isRuntimeAllowed( + LinuxContainerRuntimeConstants.RuntimeType.DEFAULT)); + } +} \ No newline at end of file