diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 81cb8c6..0f0c221 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -1495,6 +1495,10 @@ public static boolean isAclEnabled(Configuration conf) {
public static final String YARN_CONTAINER_SANDBOX_POLICY =
YARN_CONTAINER_SANDBOX + ".policy";
+ /** Prefix for group to policy file mapping*/
+ public static final String YARN_CONTAINER_SANDBOX_POLICY_GROUP_PREFIX =
+ YARN_CONTAINER_SANDBOX_POLICY + ".group.";
+
/** The group which will run by default without the java security manager.*/
public static final String YARN_CONTAINER_SANDBOX_WHITELIST_GROUP =
YARN_CONTAINER_SANDBOX + ".whitelist-group";
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/JavaSandboxLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/JavaSandboxLinuxContainerRuntime.java
index 6dc627b..38d11ef 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/JavaSandboxLinuxContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/JavaSandboxLinuxContainerRuntime.java
@@ -50,10 +50,13 @@
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.stream.Collectors;
import static org.apache.hadoop.fs.Path.SEPARATOR;
import static org.apache.hadoop.util.Shell.SYSPROP_HADOOP_HOME_DIR;
import static org.apache.hadoop.yarn.api.ApplicationConstants.Environment.JAVA_HOME;
+import static org.apache.hadoop.yarn.conf.YarnConfiguration.YARN_CONTAINER_SANDBOX;
+import static org.apache.hadoop.yarn.conf.YarnConfiguration.YARN_CONTAINER_SANDBOX_POLICY_GROUP_PREFIX;
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.CONTAINER_ID_STR;
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.CONTAINER_LOCAL_DIRS;
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.CONTAINER_RUN_CMDS;
@@ -93,13 +96,21 @@
* Accepts canonical path to a java policy file on the local filesystem.
* This file will be loaded as the base policy, any additional container
* grants will be appended to this base file. If not specified, the default
- * java.policy file provided with hadoop resources will be used.
+ * java.policy file provided with hadoop resources will be used.
*
*
* {@value YarnConfiguration#YARN_CONTAINER_SANDBOX_WHITELIST_GROUP} :
* Optional setting to specify a YARN queue which will be exempt from the
* sand-boxing process.
*
+ *
+ * {@value
+ * YarnConfiguration#YARN_CONTAINER_SANDBOX_POLICY_GROUP_PREFIX}$groupName :
+ * Optional setting to map groups to java policy files. The value is a path
+ * to the java policy file for $groupName. A user which is a member of
+ * multiple groups with different policies will receive the superset of all
+ * the permissions across their groups.
+ *
*
*/
@InterfaceAudience.Private
@@ -138,7 +149,7 @@ public void initialize(Configuration conf)
this.configuration = conf;
this.sandboxMode =
SandboxMode.get(
- this.configuration.get(YarnConfiguration.YARN_CONTAINER_SANDBOX,
+ this.configuration.get(YARN_CONTAINER_SANDBOX,
YarnConfiguration.DEFAULT_YARN_CONTAINER_SANDBOX));
initializePolicyDir();
@@ -213,8 +224,10 @@ public void prepareContainer(ContainerRuntimeContext ctx)
ctx.getExecutionAttribute(CONTAINER_RUN_CMDS);
Map env =
ctx.getContainer().getLaunchContext().getEnvironment();
+ String username =
+ ctx.getExecutionAttribute(USER);
- if(!isSandboxContainerWhitelisted(ctx, commands)) {
+ if(!isSandboxContainerWhitelisted(username, commands)) {
String tmpDirBase = configuration.get("hadoop.tmp.dir");
if (tmpDirBase == null) {
throw new ContainerExecutionException("hadoop.tmp.dir not set!");
@@ -224,6 +237,8 @@ public void prepareContainer(ContainerRuntimeContext ctx)
try {
String containerID = ctx.getExecutionAttribute(CONTAINER_ID_STR);
+ List groupPolicyFiles =
+ getGroupPolicyFiles(configuration, ctx.getExecutionAttribute(USER));
Path policyFilePath = Files.createFile(
Paths.get(policyFileDir.toString(),
containerID + "-" + NMContainerPolicyUtils.POLICY_FILE),
@@ -232,12 +247,12 @@ public void prepareContainer(ContainerRuntimeContext ctx)
containerPolicies.put(containerID, policyFilePath);
- NMContainerPolicyUtils.generatePolicyFile(
- policyOutputStream, localDirs, resources, configuration);
+ NMContainerPolicyUtils.generatePolicyFile(policyOutputStream,
+ localDirs, groupPolicyFiles, resources, configuration);
NMContainerPolicyUtils.appendSecurityFlags(
commands, env, policyFilePath, sandboxMode);
- } catch (Exception e) {
+ } catch (IOException e) {
throw new ContainerExecutionException(e);
} finally {
IOUtils.cleanup(LOG, policyOutputStream);
@@ -265,15 +280,32 @@ boolean isSandboxContainerRequested() {
return sandboxMode != SandboxMode.disabled;
}
+ private static List getGroupPolicyFiles(Configuration conf,
+ String user) throws ContainerExecutionException {
+ Groups groups = Groups.getUserToGroupsMappingService(conf);
+ List userGroups;
+ try {
+ userGroups = groups.getGroups(user);
+ } catch (IOException e) {
+ throw new ContainerExecutionException("Container user does not exist");
+ }
+
+ return userGroups.stream()
+ .map(group -> conf.get(YARN_CONTAINER_SANDBOX_POLICY_GROUP_PREFIX
+ + group))
+ .filter(groupPolicy -> groupPolicy != null)
+ .collect(Collectors.toList());
+ }
+
/**
* Determine if the container should be whitelisted (i.e. exempt from the
* Java Security Manager).
- * @param ctx The container runtime context for the requested container
+ * @param username The name of the user running the container
* @param commands The list of run commands for the container
* @return boolean value denoting whether the container should be whitelisted.
* @throws ContainerExecutionException If container user can not be resolved
*/
- private boolean isSandboxContainerWhitelisted(ContainerRuntimeContext ctx,
+ private boolean isSandboxContainerWhitelisted(String username,
List commands) throws ContainerExecutionException {
String whitelistGroup = configuration.get(
YarnConfiguration.YARN_CONTAINER_SANDBOX_WHITELIST_GROUP);
@@ -282,7 +314,7 @@ private boolean isSandboxContainerWhitelisted(ContainerRuntimeContext ctx,
boolean isWhitelisted = false;
try {
- userGroups = groups.getGroups(ctx.getExecutionAttribute(USER));
+ userGroups = groups.getGroups(username);
} catch (IOException e) {
throw new ContainerExecutionException("Container user does not exist");
}
@@ -396,8 +428,9 @@ public String toString(){
* base policy file or if it is unable to create a new policy file.
*/
static void generatePolicyFile(OutputStream policyOutStream,
- List localDirs, Map> resources, Configuration conf)
+ List localDirs, List groupPolicyPaths,
+ Map> resources,
+ Configuration conf)
throws IOException {
String policyFilePath =
@@ -411,13 +444,17 @@ static void generatePolicyFile(OutputStream policyOutStream,
cacheDirs.add(path.getParent().toString());
}
+ if(groupPolicyPaths != null){
+ for(String policyPath : groupPolicyPaths){
+ Files.copy(Paths.get(policyPath), policyOutStream);
+ }
+ }
if(policyFilePath == null) {
IOUtils.copyBytes(
NMContainerPolicyUtils.class.getResourceAsStream("/" + POLICY_FILE),
policyOutStream, conf, false);
} else {
Files.copy(Paths.get(policyFilePath), policyOutStream);
- policyOutStream.flush();
}
Formatter filePermissionFormat = new Formatter(policyOutStream,
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestJavaSandboxLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestJavaSandboxLinuxContainerRuntime.java
index e482c8d..c82d4e0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestJavaSandboxLinuxContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestJavaSandboxLinuxContainerRuntime.java
@@ -21,6 +21,7 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
@@ -37,19 +38,27 @@
import java.io.File;
import java.io.FileOutputStream;
import java.io.FilePermission;
+import java.io.FileWriter;
+import java.io.IOException;
import java.io.OutputStream;
+import java.net.SocketPermission;
import java.nio.file.Files;
import java.nio.file.Paths;
+import java.security.AccessControlException;
+import java.security.Permission;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
import static org.apache.hadoop.yarn.api.ApplicationConstants.Environment.JAVA_HOME;
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils.CHAINED_COMMAND_REGEX;
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils.CLEAN_CMD_REGEX;
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils.CONTAINS_JAVA_CMD;
+import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils.POLICY_APPEND_FLAG;
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils.POLICY_FILE;
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils.POLICY_FLAG;
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils.SECURITY_FLAG;
@@ -100,11 +109,12 @@
private final static String WHITELIST_GROUP = "captains";
private final static String CONTAINER_ID = "container_1234567890";
private final static String APPLICATION_ID = "application_1234567890";
+ private File baseTestDirectory;
@Before
public void setup() throws Exception {
- File baseTestDirectory = new File(System.getProperty("test.build.data",
+ baseTestDirectory = new File(System.getProperty("test.build.data",
System.getProperty("java.io.tmpdir", "target")),
TestJavaSandboxLinuxContainerRuntime.class.getName());
@@ -115,8 +125,6 @@ public void setup() throws Exception {
conf.set(CommonConfigurationKeys.HADOOP_USER_GROUP_STATIC_OVERRIDES,
WHITELIST_USER + "=" + WHITELIST_GROUP + ";"
+ NORMAL_USER + "=" + NORMAL_GROUP + ";");
- conf.set(YarnConfiguration.YARN_CONTAINER_SANDBOX_WHITELIST_GROUP,
- WHITELIST_GROUP);
conf.set("hadoop.tmp.dir", baseTestDirectory.getAbsolutePath());
Files.deleteIfExists(Paths.get(baseTestDirectory.getAbsolutePath(),
@@ -156,7 +164,7 @@ public void setup() throws Exception {
OutputStream outStream = new FileOutputStream(policyFile);
JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils
- .generatePolicyFile(outStream, symLinks, resources, conf);
+ .generatePolicyFile(outStream, symLinks, null, resources, conf);
outStream.close();
System.setProperty("java.security.policy", policyFile.getCanonicalPath());
@@ -164,7 +172,7 @@ public void setup() throws Exception {
}
- public ContainerRuntimeContext.Builder createRuntimeContext(){
+ public ContainerRuntimeContext.Builder createRuntimeContext(){
Container container = mock(Container.class);
ContainerLaunchContext ctx = mock(ContainerLaunchContext.class);
@@ -194,6 +202,44 @@ public void setup() throws Exception {
}
@Test
+ public void testGroupPolicy() throws IOException, ContainerExecutionException{
+ // Generate new policy file which contains grant
+ File restrictedPolicyFile =
+ File.createTempFile("openSocket", "policy", baseTestDirectory);
+ Permission perm = new SocketPermission("localhost:0", "listen");
+ Permission runtimePerm = new RuntimePermission("createClassLoader");
+
+ FileWriter permissivePolicyWriter =
+ new FileWriter(restrictedPolicyFile);
+ permissivePolicyWriter.write("grant { "+ perm.toString() +"\n };");
+ IOUtils.cleanup(null, permissivePolicyWriter);
+
+ conf.set(YarnConfiguration.YARN_CONTAINER_SANDBOX_POLICY_GROUP_PREFIX +
+ WHITELIST_GROUP, restrictedPolicyFile.toString());
+
+ String[] inputCommand = {"$JAVA_HOME/bin/java jar MyJob.jar"};
+ List commands = Arrays.asList(inputCommand);
+
+ runtimeContextBuilder.setExecutionAttribute(USER, WHITELIST_USER);
+ runtimeContextBuilder.setExecutionAttribute(CONTAINER_RUN_CMDS, commands);
+
+ runtime.prepareContainer(runtimeContextBuilder.build());
+
+ //pull generated policy from cmd
+ Matcher policyMatches = Pattern.compile(POLICY_APPEND_FLAG + "=?([^ ]+)")
+ .matcher(commands.get(0));
+ policyMatches.find();
+ String generatedPolicy = policyMatches.group();
+
+ //Test that generated policy file has included permissive policy
+ System.setProperty("java.security.policy", generatedPolicy);
+ SecurityManager secman = new SecurityManager();
+ secman.checkPermission(perm);
+ exception.expect(AccessControlException.class);
+ secman.checkPermission(runtimePerm);
+ }
+
+ @Test
public void testGrant() throws Exception {
FilePermission grantPermission =
new FilePermission(grantFile.getAbsolutePath(), "read");
@@ -246,6 +292,9 @@ public void testDisabledSandboxWithWhitelist()
};
List commands = Arrays.asList(inputCommand);
+ conf.set(YarnConfiguration.YARN_CONTAINER_SANDBOX_WHITELIST_GROUP,
+ WHITELIST_GROUP);
+
runtimeContextBuilder.setExecutionAttribute(USER, WHITELIST_USER);
runtimeContextBuilder.setExecutionAttribute(CONTAINER_RUN_CMDS, commands);
runtime.prepareContainer(runtimeContextBuilder.build());
@@ -264,6 +313,9 @@ public void testEnabledSandboxWithWhitelist()
};
List commands = Arrays.asList(inputCommand);
+ conf.set(YarnConfiguration.YARN_CONTAINER_SANDBOX_WHITELIST_GROUP,
+ WHITELIST_GROUP);
+
runtimeContextBuilder.setExecutionAttribute(USER, WHITELIST_USER);
runtimeContextBuilder.setExecutionAttribute(CONTAINER_RUN_CMDS, commands);
runtime.prepareContainer(runtimeContextBuilder.build());
@@ -282,6 +334,9 @@ public void testDeniedWhitelistGroup() throws ContainerExecutionException {
};
List commands = Arrays.asList(inputCommand);
+ conf.set(YarnConfiguration.YARN_CONTAINER_SANDBOX_WHITELIST_GROUP,
+ WHITELIST_GROUP);
+
runtimeContextBuilder.setExecutionAttribute(USER, NORMAL_USER);
runtimeContextBuilder.setExecutionAttribute(CONTAINER_RUN_CMDS, commands);
runtime.prepareContainer(runtimeContextBuilder.build());