diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 81cb8c6..0f0c221 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -1495,6 +1495,10 @@ public static boolean isAclEnabled(Configuration conf) { public static final String YARN_CONTAINER_SANDBOX_POLICY = YARN_CONTAINER_SANDBOX + ".policy"; + /** Prefix for group to policy file mapping*/ + public static final String YARN_CONTAINER_SANDBOX_POLICY_GROUP_PREFIX = + YARN_CONTAINER_SANDBOX_POLICY + ".group."; + /** The group which will run by default without the java security manager.*/ public static final String YARN_CONTAINER_SANDBOX_WHITELIST_GROUP = YARN_CONTAINER_SANDBOX + ".whitelist-group"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/JavaSandboxLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/JavaSandboxLinuxContainerRuntime.java index 6dc627b..38d11ef 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/JavaSandboxLinuxContainerRuntime.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/JavaSandboxLinuxContainerRuntime.java @@ -50,10 +50,13 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.stream.Collectors; import static org.apache.hadoop.fs.Path.SEPARATOR; import static org.apache.hadoop.util.Shell.SYSPROP_HADOOP_HOME_DIR; import static org.apache.hadoop.yarn.api.ApplicationConstants.Environment.JAVA_HOME; +import static org.apache.hadoop.yarn.conf.YarnConfiguration.YARN_CONTAINER_SANDBOX; +import static org.apache.hadoop.yarn.conf.YarnConfiguration.YARN_CONTAINER_SANDBOX_POLICY_GROUP_PREFIX; import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.CONTAINER_ID_STR; import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.CONTAINER_LOCAL_DIRS; import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.CONTAINER_RUN_CMDS; @@ -93,13 +96,21 @@ * Accepts canonical path to a java policy file on the local filesystem. * This file will be loaded as the base policy, any additional container * grants will be appended to this base file. If not specified, the default - * java.policy file provided with hadoop resources will be used. + * java.policy file provided with hadoop resources will be used. * *
  • * {@value YarnConfiguration#YARN_CONTAINER_SANDBOX_WHITELIST_GROUP} : * Optional setting to specify a YARN queue which will be exempt from the * sand-boxing process. *
  • + *
  • + * {@value + * YarnConfiguration#YARN_CONTAINER_SANDBOX_POLICY_GROUP_PREFIX}$groupName : + * Optional setting to map groups to java policy files. The value is a path + * to the java policy file for $groupName. A user which is a member of + * multiple groups with different policies will receive the superset of all + * the permissions across their groups. + *
  • * */ @InterfaceAudience.Private @@ -138,7 +149,7 @@ public void initialize(Configuration conf) this.configuration = conf; this.sandboxMode = SandboxMode.get( - this.configuration.get(YarnConfiguration.YARN_CONTAINER_SANDBOX, + this.configuration.get(YARN_CONTAINER_SANDBOX, YarnConfiguration.DEFAULT_YARN_CONTAINER_SANDBOX)); initializePolicyDir(); @@ -213,8 +224,10 @@ public void prepareContainer(ContainerRuntimeContext ctx) ctx.getExecutionAttribute(CONTAINER_RUN_CMDS); Map env = ctx.getContainer().getLaunchContext().getEnvironment(); + String username = + ctx.getExecutionAttribute(USER); - if(!isSandboxContainerWhitelisted(ctx, commands)) { + if(!isSandboxContainerWhitelisted(username, commands)) { String tmpDirBase = configuration.get("hadoop.tmp.dir"); if (tmpDirBase == null) { throw new ContainerExecutionException("hadoop.tmp.dir not set!"); @@ -224,6 +237,8 @@ public void prepareContainer(ContainerRuntimeContext ctx) try { String containerID = ctx.getExecutionAttribute(CONTAINER_ID_STR); + List groupPolicyFiles = + getGroupPolicyFiles(configuration, ctx.getExecutionAttribute(USER)); Path policyFilePath = Files.createFile( Paths.get(policyFileDir.toString(), containerID + "-" + NMContainerPolicyUtils.POLICY_FILE), @@ -232,12 +247,12 @@ public void prepareContainer(ContainerRuntimeContext ctx) containerPolicies.put(containerID, policyFilePath); - NMContainerPolicyUtils.generatePolicyFile( - policyOutputStream, localDirs, resources, configuration); + NMContainerPolicyUtils.generatePolicyFile(policyOutputStream, + localDirs, groupPolicyFiles, resources, configuration); NMContainerPolicyUtils.appendSecurityFlags( commands, env, policyFilePath, sandboxMode); - } catch (Exception e) { + } catch (IOException e) { throw new ContainerExecutionException(e); } finally { IOUtils.cleanup(LOG, policyOutputStream); @@ -265,15 +280,32 @@ boolean isSandboxContainerRequested() { return sandboxMode != SandboxMode.disabled; } + private static List getGroupPolicyFiles(Configuration conf, + String user) throws ContainerExecutionException { + Groups groups = Groups.getUserToGroupsMappingService(conf); + List userGroups; + try { + userGroups = groups.getGroups(user); + } catch (IOException e) { + throw new ContainerExecutionException("Container user does not exist"); + } + + return userGroups.stream() + .map(group -> conf.get(YARN_CONTAINER_SANDBOX_POLICY_GROUP_PREFIX + + group)) + .filter(groupPolicy -> groupPolicy != null) + .collect(Collectors.toList()); + } + /** * Determine if the container should be whitelisted (i.e. exempt from the * Java Security Manager). - * @param ctx The container runtime context for the requested container + * @param username The name of the user running the container * @param commands The list of run commands for the container * @return boolean value denoting whether the container should be whitelisted. * @throws ContainerExecutionException If container user can not be resolved */ - private boolean isSandboxContainerWhitelisted(ContainerRuntimeContext ctx, + private boolean isSandboxContainerWhitelisted(String username, List commands) throws ContainerExecutionException { String whitelistGroup = configuration.get( YarnConfiguration.YARN_CONTAINER_SANDBOX_WHITELIST_GROUP); @@ -282,7 +314,7 @@ private boolean isSandboxContainerWhitelisted(ContainerRuntimeContext ctx, boolean isWhitelisted = false; try { - userGroups = groups.getGroups(ctx.getExecutionAttribute(USER)); + userGroups = groups.getGroups(username); } catch (IOException e) { throw new ContainerExecutionException("Container user does not exist"); } @@ -396,8 +428,9 @@ public String toString(){ * base policy file or if it is unable to create a new policy file. */ static void generatePolicyFile(OutputStream policyOutStream, - List localDirs, Map> resources, Configuration conf) + List localDirs, List groupPolicyPaths, + Map> resources, + Configuration conf) throws IOException { String policyFilePath = @@ -411,13 +444,17 @@ static void generatePolicyFile(OutputStream policyOutStream, cacheDirs.add(path.getParent().toString()); } + if(groupPolicyPaths != null){ + for(String policyPath : groupPolicyPaths){ + Files.copy(Paths.get(policyPath), policyOutStream); + } + } if(policyFilePath == null) { IOUtils.copyBytes( NMContainerPolicyUtils.class.getResourceAsStream("/" + POLICY_FILE), policyOutStream, conf, false); } else { Files.copy(Paths.get(policyFilePath), policyOutStream); - policyOutStream.flush(); } Formatter filePermissionFormat = new Formatter(policyOutStream, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestJavaSandboxLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestJavaSandboxLinuxContainerRuntime.java index e482c8d..c82d4e0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestJavaSandboxLinuxContainerRuntime.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestJavaSandboxLinuxContainerRuntime.java @@ -21,6 +21,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; @@ -37,19 +38,27 @@ import java.io.File; import java.io.FileOutputStream; import java.io.FilePermission; +import java.io.FileWriter; +import java.io.IOException; import java.io.OutputStream; +import java.net.SocketPermission; import java.nio.file.Files; import java.nio.file.Paths; +import java.security.AccessControlException; +import java.security.Permission; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import static org.apache.hadoop.yarn.api.ApplicationConstants.Environment.JAVA_HOME; import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils.CHAINED_COMMAND_REGEX; import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils.CLEAN_CMD_REGEX; import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils.CONTAINS_JAVA_CMD; +import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils.POLICY_APPEND_FLAG; import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils.POLICY_FILE; import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils.POLICY_FLAG; import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils.SECURITY_FLAG; @@ -100,11 +109,12 @@ private final static String WHITELIST_GROUP = "captains"; private final static String CONTAINER_ID = "container_1234567890"; private final static String APPLICATION_ID = "application_1234567890"; + private File baseTestDirectory; @Before public void setup() throws Exception { - File baseTestDirectory = new File(System.getProperty("test.build.data", + baseTestDirectory = new File(System.getProperty("test.build.data", System.getProperty("java.io.tmpdir", "target")), TestJavaSandboxLinuxContainerRuntime.class.getName()); @@ -115,8 +125,6 @@ public void setup() throws Exception { conf.set(CommonConfigurationKeys.HADOOP_USER_GROUP_STATIC_OVERRIDES, WHITELIST_USER + "=" + WHITELIST_GROUP + ";" + NORMAL_USER + "=" + NORMAL_GROUP + ";"); - conf.set(YarnConfiguration.YARN_CONTAINER_SANDBOX_WHITELIST_GROUP, - WHITELIST_GROUP); conf.set("hadoop.tmp.dir", baseTestDirectory.getAbsolutePath()); Files.deleteIfExists(Paths.get(baseTestDirectory.getAbsolutePath(), @@ -156,7 +164,7 @@ public void setup() throws Exception { OutputStream outStream = new FileOutputStream(policyFile); JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils - .generatePolicyFile(outStream, symLinks, resources, conf); + .generatePolicyFile(outStream, symLinks, null, resources, conf); outStream.close(); System.setProperty("java.security.policy", policyFile.getCanonicalPath()); @@ -164,7 +172,7 @@ public void setup() throws Exception { } - public ContainerRuntimeContext.Builder createRuntimeContext(){ + public ContainerRuntimeContext.Builder createRuntimeContext(){ Container container = mock(Container.class); ContainerLaunchContext ctx = mock(ContainerLaunchContext.class); @@ -194,6 +202,44 @@ public void setup() throws Exception { } @Test + public void testGroupPolicy() throws IOException, ContainerExecutionException{ + // Generate new policy file which contains grant + File restrictedPolicyFile = + File.createTempFile("openSocket", "policy", baseTestDirectory); + Permission perm = new SocketPermission("localhost:0", "listen"); + Permission runtimePerm = new RuntimePermission("createClassLoader"); + + FileWriter permissivePolicyWriter = + new FileWriter(restrictedPolicyFile); + permissivePolicyWriter.write("grant { "+ perm.toString() +"\n };"); + IOUtils.cleanup(null, permissivePolicyWriter); + + conf.set(YarnConfiguration.YARN_CONTAINER_SANDBOX_POLICY_GROUP_PREFIX + + WHITELIST_GROUP, restrictedPolicyFile.toString()); + + String[] inputCommand = {"$JAVA_HOME/bin/java jar MyJob.jar"}; + List commands = Arrays.asList(inputCommand); + + runtimeContextBuilder.setExecutionAttribute(USER, WHITELIST_USER); + runtimeContextBuilder.setExecutionAttribute(CONTAINER_RUN_CMDS, commands); + + runtime.prepareContainer(runtimeContextBuilder.build()); + + //pull generated policy from cmd + Matcher policyMatches = Pattern.compile(POLICY_APPEND_FLAG + "=?([^ ]+)") + .matcher(commands.get(0)); + policyMatches.find(); + String generatedPolicy = policyMatches.group(); + + //Test that generated policy file has included permissive policy + System.setProperty("java.security.policy", generatedPolicy); + SecurityManager secman = new SecurityManager(); + secman.checkPermission(perm); + exception.expect(AccessControlException.class); + secman.checkPermission(runtimePerm); + } + + @Test public void testGrant() throws Exception { FilePermission grantPermission = new FilePermission(grantFile.getAbsolutePath(), "read"); @@ -246,6 +292,9 @@ public void testDisabledSandboxWithWhitelist() }; List commands = Arrays.asList(inputCommand); + conf.set(YarnConfiguration.YARN_CONTAINER_SANDBOX_WHITELIST_GROUP, + WHITELIST_GROUP); + runtimeContextBuilder.setExecutionAttribute(USER, WHITELIST_USER); runtimeContextBuilder.setExecutionAttribute(CONTAINER_RUN_CMDS, commands); runtime.prepareContainer(runtimeContextBuilder.build()); @@ -264,6 +313,9 @@ public void testEnabledSandboxWithWhitelist() }; List commands = Arrays.asList(inputCommand); + conf.set(YarnConfiguration.YARN_CONTAINER_SANDBOX_WHITELIST_GROUP, + WHITELIST_GROUP); + runtimeContextBuilder.setExecutionAttribute(USER, WHITELIST_USER); runtimeContextBuilder.setExecutionAttribute(CONTAINER_RUN_CMDS, commands); runtime.prepareContainer(runtimeContextBuilder.build()); @@ -282,6 +334,9 @@ public void testDeniedWhitelistGroup() throws ContainerExecutionException { }; List commands = Arrays.asList(inputCommand); + conf.set(YarnConfiguration.YARN_CONTAINER_SANDBOX_WHITELIST_GROUP, + WHITELIST_GROUP); + runtimeContextBuilder.setExecutionAttribute(USER, NORMAL_USER); runtimeContextBuilder.setExecutionAttribute(CONTAINER_RUN_CMDS, commands); runtime.prepareContainer(runtimeContextBuilder.build());