Index: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandlerImpl.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandlerImpl.java (revision 615ac09499dc0b391cbb99bb0e9877959a9173a6) +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandlerImpl.java (revision 33a95b9e6d6ee99c1165d6f4bfc0c8d4ab0bc81d) @@ -21,12 +21,14 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources; import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Joiner; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperation; @@ -40,6 +42,8 @@ import java.nio.file.Paths; import java.util.Arrays; import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.concurrent.locks.ReadWriteLock; @@ -66,6 +70,7 @@ private final long deleteCGroupTimeout; private final long deleteCGroupDelay; private Map controllerPaths; + private Map> parsedMtab; private final ReadWriteLock rwLock; private final PrivilegedOperationExecutor privilegedOperationExecutor; private final Clock clock; @@ -95,6 +100,7 @@ conf.getLong(YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_DELETE_DELAY, YarnConfiguration.DEFAULT_NM_LINUX_CONTAINER_CGROUPS_DELETE_DELAY); this.controllerPaths = new HashMap<>(); + this.parsedMtab = new HashMap<>(); this.rwLock = new ReentrantReadWriteLock(); this.privilegedOperationExecutor = privilegedOperationExecutor; this.clock = SystemClock.getInstance(); @@ -128,51 +134,53 @@ } private void initializeControllerPaths() throws ResourceHandlerException { - if (enableCGroupMount) { - // nothing to do here - we support 'deferred' mounting of specific - // controllers - we'll populate the path for a given controller when an - // explicit mountCGroupController request is issued. - LOG.info("CGroup controller mounting enabled."); - } else { - // cluster admins are expected to have mounted controllers in specific - // locations - we'll attempt to figure out mount points + // Cluster admins may have some subsystems mounted in specific locations + // We'll attempt to figure out mount points. We do this even if we plan + // to mount cgroups into our own tree to control the path permissions or + // to mount subsystems that are not mounted previously. + // The subsystems for new and existing mount points have to match, and + // the same hierarchy will be mounted at each mount point with the same + // subsystem set. - Map cPaths = - initializeControllerPathsFromMtab(mtabFile, this.cGroupPrefix); - // we want to do a bulk update without the paths changing concurrently - try { - rwLock.writeLock().lock(); - controllerPaths = cPaths; - } finally { - rwLock.writeLock().unlock(); - } + Map> newMtab; + Map cPaths; + try { + // parse mtab + newMtab = parseMtab(mtabFile); + + // find cgroup controller paths + cPaths = initializeControllerPathsFromMtab(newMtab); + } catch (IOException e) { + LOG.warn("Failed to initialize controller paths! Exception: " + e); + throw new ResourceHandlerException( + "Failed to initialize controller paths!"); + } + + // we want to do a bulk update without the paths changing concurrently + try { + rwLock.writeLock().lock(); + controllerPaths = cPaths; + parsedMtab = newMtab; + } finally { + rwLock.writeLock().unlock(); } } @VisibleForTesting static Map initializeControllerPathsFromMtab( - String mtab, String cGroupPrefix) throws ResourceHandlerException { - try { - Map> parsedMtab = parseMtab(mtab); - Map ret = new HashMap<>(); + Map> parsedMtab) + throws ResourceHandlerException { + Map ret = new HashMap<>(); - for (CGroupController controller : CGroupController.values()) { - String subsystemName = controller.getName(); - String controllerPath = findControllerInMtab(subsystemName, parsedMtab); + for (CGroupController controller : CGroupController.values()) { + String subsystemName = controller.getName(); + String controllerPath = findControllerInMtab(subsystemName, parsedMtab); - if (controllerPath != null) { - ret.put(controller, controllerPath); - } else { - LOG.warn("Controller not mounted but automount disabled: " + - subsystemName); - } - } - return ret; - } catch (IOException e) { - LOG.warn("Failed to initialize controller paths! Exception: " + e); - throw new ResourceHandlerException( - "Failed to initialize controller paths!"); - } + if (controllerPath != null) { + ret.put(controller, controllerPath); + } + } + return ret; } /* We are looking for entries of the form: @@ -190,10 +198,15 @@ * for mounts with type "cgroup". Cgroup controllers will * appear in the list of options for a path. */ - private static Map> parseMtab(String mtab) + @VisibleForTesting + static Map> parseMtab(String mtab) throws IOException { - Map> ret = new HashMap>(); + Map> ret = new HashMap<>(); BufferedReader in = null; + HashSet validCgroups = new HashSet<>(); + for (CGroupController controller : CGroupController.values()) { + validCgroups.add(controller.getName()); + } try { FileInputStream fis = new FileInputStream(new File(mtab)); @@ -209,8 +222,15 @@ String options = m.group(3); if (type.equals(CGROUPS_FSTYPE)) { - List value = Arrays.asList(options.split(",")); - ret.put(path, value); + List optionsList = Arrays.asList(options.split(",")); + List cgroupList = new LinkedList<>(); + // Collect the valid subsystem names + for(String cgroup: optionsList) { + if (validCgroups.contains(cgroup)) { + cgroupList.add(cgroup); + } + } + ret.put(path, cgroupList); } } } @@ -245,31 +265,43 @@ private void mountCGroupController(CGroupController controller) throws ResourceHandlerException { - String path = getControllerPath(controller); + if (cGroupMountPath == null) { + throw new ResourceHandlerException("Trying to mount to null mount path"); + } + String existingMountPath = getControllerPath(controller); + String requestedMountPath = + new File(cGroupMountPath, controller.getName()).getAbsolutePath(); - if (path == null) { + if (existingMountPath == null || + !requestedMountPath.equals(existingMountPath)) { try { //lock out other readers/writers till we are done rwLock.writeLock().lock(); - String hierarchy = cGroupPrefix; - StringBuffer controllerPath = new StringBuffer() - .append(cGroupMountPath).append('/').append(controller.getName()); - StringBuffer cGroupKV = new StringBuffer() - .append(controller.getName()).append('=').append(controllerPath); + // If the controller was already mounted we have to mount it + // with the same options to clone the mount point otherwise + // the operation will fail + String mountOptions; + if (existingMountPath != null) { + mountOptions = Joiner.on(',') + .join(parsedMtab.get(existingMountPath)); + } else { + mountOptions = controller.getName(); + } + + String cGroupKV = + mountOptions + "=" + requestedMountPath; PrivilegedOperation.OperationType opType = PrivilegedOperation .OperationType.MOUNT_CGROUPS; PrivilegedOperation op = new PrivilegedOperation(opType); - op.appendArgs(hierarchy, cGroupKV.toString()); + op.appendArgs(cGroupPrefix, cGroupKV); LOG.info("Mounting controller " + controller.getName() + " at " + - controllerPath); + requestedMountPath); privilegedOperationExecutor.executePrivilegedOperation(op, false); //if privileged operation succeeds, update controller paths - controllerPaths.put(controller, controllerPath.toString()); - - return; + controllerPaths.put(controller, requestedMountPath); } catch (PrivilegedOperationException e) { LOG.error("Failed to mount controller: " + controller.getName()); throw new ResourceHandlerException("Failed to mount controller: " @@ -278,37 +310,34 @@ rwLock.writeLock().unlock(); } } else { - LOG.info("CGroup controller already mounted at: " + path); - return; + LOG.info("CGroup controller already mounted at: " + existingMountPath); } } @Override public String getRelativePathForCGroup(String cGroupId) { - return new StringBuffer(cGroupPrefix).append("/") - .append(cGroupId).toString(); + return cGroupPrefix + Path.SEPARATOR + cGroupId; } @Override public String getPathForCGroup(CGroupController controller, String cGroupId) { - return new StringBuffer(getControllerPath(controller)) - .append('/').append(cGroupPrefix).append("/") - .append(cGroupId).toString(); + return getControllerPath(controller) + Path.SEPARATOR + cGroupPrefix + + Path.SEPARATOR + cGroupId; } @Override public String getPathForCGroupTasks(CGroupController controller, String cGroupId) { - return new StringBuffer(getPathForCGroup(controller, cGroupId)) - .append('/').append(CGROUP_FILE_TASKS).toString(); + return getPathForCGroup(controller, cGroupId) + + Path.SEPARATOR + CGROUP_FILE_TASKS; } @Override public String getPathForCGroupParam(CGroupController controller, String cGroupId, String param) { - return new StringBuffer(getPathForCGroup(controller, cGroupId)) - .append('/').append(controller.getName()).append('.') - .append(param).toString(); + return getPathForCGroup(controller, cGroupId) + + Path.SEPARATOR + controller.getName() + + "." + param; } /** @@ -324,10 +353,18 @@ // We have a controller that needs to be mounted mountCGroupController(controller); } else { - // We are working with a pre-mounted contoller - // Make sure that Yarn cgroup hierarchy path exists - initializePreMountedCGroupController(controller); - } + String controllerPath = getControllerPath(controller); + + if (controllerPath == null) { + throw new ResourceHandlerException( + "Controller not mounted but automount disabled: " + + controller.getName()); + } + } + + // We are working with a pre-mounted contoller + // Make sure that Yarn cgroup hierarchy path exists + initializePreMountedCGroupController(controller); } /** @@ -341,11 +378,18 @@ * @throws ResourceHandlerException yarn hierarchy cannot be created or * accessed for any reason */ - public void initializePreMountedCGroupController(CGroupController controller) + private void initializePreMountedCGroupController(CGroupController controller) throws ResourceHandlerException { // Check permissions to cgroup hierarchy and // create YARN cgroup if it does not exist, yet - File rootHierarchy = new File(getControllerPath(controller)); + String controllerPath = getControllerPath(controller); + + if (controllerPath == null) { + throw new ResourceHandlerException( + "Controller not mounted: " + controller.getName()); + } + + File rootHierarchy = new File(controllerPath); File yarnHierarchy = new File(rootHierarchy, cGroupPrefix); String subsystemName = controller.getName(); @@ -397,17 +441,10 @@ String errorMessage, String subsystemName, String yarnCgroupPath) { - return new StringBuilder() - .append(errorMessage) - .append(" Subsystem:") - .append(subsystemName) - .append(" Mount points:") - .append(mtabFile) - .append(" User:") - .append(System.getProperty("user.name")) - .append(" Path: ") - .append(yarnCgroupPath) - .toString(); + return errorMessage + " Subsystem:" + subsystemName + + " Mount points:" + mtabFile + + " User:" + System.getProperty("user.name") + + " Path: " + yarnCgroupPath; } @Override @@ -450,7 +487,7 @@ * @param cgf object referring to the cgroup to be deleted * @return Boolean indicating whether cgroup was deleted */ - boolean checkAndDeleteCgroup(File cgf) throws InterruptedException { + private boolean checkAndDeleteCgroup(File cgf) throws InterruptedException { boolean deleted = false; // FileInputStream in = null; try (FileInputStream in = new FileInputStream(cgf + "/tasks")) { @@ -508,6 +545,7 @@ String param, String value) throws ResourceHandlerException { String cGroupParamPath = getPathForCGroupParam(controller, cGroupId, param); PrintWriter pw = null; + ResourceHandlerException exceptionToThrow = null; if (LOG.isDebugEnabled()) { LOG.debug( @@ -521,25 +559,28 @@ pw = new PrintWriter(w); pw.write(value); } catch (IOException e) { - throw new ResourceHandlerException(new StringBuffer("Unable to write to ") - .append(cGroupParamPath).append(" with value: ").append(value) - .toString(), e); + throw new ResourceHandlerException( + "Unable to write to " + cGroupParamPath + + " with value: " + value, e); } finally { if (pw != null) { boolean hasError = pw.checkError(); pw.close(); if (hasError) { - throw new ResourceHandlerException( - new StringBuffer("Unable to write to ") - .append(cGroupParamPath).append(" with value: ").append(value) - .toString()); + exceptionToThrow = new ResourceHandlerException( + "Unable to write to " + cGroupParamPath + + " with value: " + value); } if (pw.checkError()) { - throw new ResourceHandlerException("Error while closing cgroup file" + + exceptionToThrow = new ResourceHandlerException( + "Error while closing cgroup file" + " " + cGroupParamPath); } } } + if (exceptionToThrow != null) { + throw exceptionToThrow; + } } @Override Index: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c (revision 615ac09499dc0b391cbb99bb0e9877959a9173a6) +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c (revision 3a1f618355ea8672db9ca742513af2efe040145d) @@ -2046,17 +2046,31 @@ fprintf(LOGFILE, "Failed to mount cgroup controller, not supported\n"); return -1; #else - char *controller = malloc(strlen(pair)); - char *mount_path = malloc(strlen(pair)); + size_t len = strlen(pair); + char *controller = malloc(len); + char *mount_path = malloc(len); char hier_path[EXECUTOR_PATH_MAX]; int result = 0; + struct stat sb; - if (get_kv_key(pair, controller, strlen(pair)) < 0 || - get_kv_value(pair, mount_path, strlen(pair)) < 0) { + if (controller == NULL || mount_path == NULL) { + fprintf(LOGFILE, "Failed to mount cgroup controller; not enough memory\n"); + result = OUT_OF_MEMORY; + } + if (get_kv_key(pair, controller, len) < 0 || + get_kv_value(pair, mount_path, len) < 0) { fprintf(LOGFILE, "Failed to mount cgroup controller; invalid option: %s\n", pair); result = -1; } else { + if (stat(mount_path, &sb) != 0) { + // Create mount point, if it does not exist + const mode_t mount_perms = S_IRWXU | S_IRGRP | S_IXGRP; + if (mkdirs(mount_path, mount_perms) == 0) { + fprintf(LOGFILE, "Failed to create cgroup mount point %s at %s\n", + controller, mount_path); + } + } if (mount("none", mount_path, "cgroup", 0, controller) == 0) { char *buf = stpncpy(hier_path, mount_path, strlen(mount_path)); *buf++ = '/'; Index: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsHandlerImpl.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsHandlerImpl.java (revision 615ac09499dc0b391cbb99bb0e9877959a9173a6) +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsHandlerImpl.java (revision 33a95b9e6d6ee99c1165d6f4bfc0c8d4ab0bc81d) @@ -25,6 +25,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperation; import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationException; @@ -40,9 +41,11 @@ import java.io.IOException; import java.nio.file.Files; import java.security.Permission; +import java.util.List; import java.util.Map; import java.util.UUID; +import static org.junit.Assert.assertTrue; import static org.mockito.Matchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; @@ -57,7 +60,6 @@ LogFactory.getLog(TestCGroupsHandlerImpl.class); private PrivilegedOperationExecutor privilegedOperationExecutorMock; - private Configuration conf; private String tmpPath; private String hierarchy; private CGroupsHandler.CGroupController controller; @@ -66,36 +68,135 @@ @Before public void setup() { privilegedOperationExecutorMock = mock(PrivilegedOperationExecutor.class); - conf = new YarnConfiguration(); + + // Prepare test directory tmpPath = System.getProperty("test.build.data") + "/cgroups"; + File tmpDir = new File(tmpPath); + FileUtils.deleteQuietly(tmpDir); + assertTrue(tmpDir.mkdirs()); + //no leading or trailing slashes here hierarchy = "test-hadoop-yarn"; + // Sample subsystem. Not used by all the tests + controller = CGroupsHandler.CGroupController.NET_CLS; + controllerPath = + new File(new File(tmpPath, controller.getName()), hierarchy) + .getAbsolutePath(); + } + + @After + public void teardown() { + FileUtil.fullyDelete(new File(tmpPath)); + } + + /** + * Security manager simulating access denied + */ + private class MockSecurityManagerDenyWrite extends SecurityManager { + @Override + public void checkPermission(Permission perm) { + if(perm.getActions().equals("write")) { + throw new SecurityException("Mock not allowed"); + } + } + } + + /** + * Create configuration to mount cgroups that do not exist + * @return configuration object + */ + private YarnConfiguration createMountConfiguration() { + YarnConfiguration conf = new YarnConfiguration(); conf.set(YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_HIERARCHY, hierarchy); conf.setBoolean(YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_MOUNT, true); conf.set(YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_MOUNT_PATH, tmpPath); - controller = CGroupsHandler.CGroupController.NET_CLS; - controllerPath = new StringBuffer(tmpPath).append('/') - .append(controller.getName()).append('/').append(hierarchy).toString(); + return conf; + } + + /** + * Create configuration where the cgroups are premounted + * @param myHierarchy Yarn cgroup + * @return configuration object + */ + private Configuration createNoMountConfiguration(String myHierarchy) { + Configuration confNoMount = new Configuration(); + confNoMount.set(YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_HIERARCHY, + myHierarchy); + confNoMount.setBoolean(YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_MOUNT, + false); + return confNoMount; + } + + /** + * Create an empty mtab file. No cgroups are premounted + * @return mtab file + * @throws IOException could not create file + */ + private File createEmptyCgroups() throws IOException { + File emptyMtab = new File(tmpPath, "mtab"); + assertTrue("New file should have been created", emptyMtab.createNewFile()); + return emptyMtab; + } + + /** + * Create simulated cgroups mount point + * @param parentDir cgroups mount point + * @param cpuAcct simulate newer Linux behavior by mounting cpu with cpuacct + * @return simulated mtab file location + * @throws IOException mtab file was not created + */ + public static File createPremountedCgroups(File parentDir, boolean cpuAcct) + throws IOException { + File cpuCgroup = new File(parentDir, "cpu"); + String cpuMtabContent = + "none " + cpuCgroup.getAbsolutePath() + + " cgroup rw,relatime,cpu" + + (cpuAcct ? ",cpuacct" :"") + + " 0 0\n"; + assertTrue("Directory should be created", cpuCgroup.mkdirs()); + + File blkioCgroup = new File(parentDir, "blkio"); + String blkioMtabContent = + "none " + blkioCgroup.getAbsolutePath() + + " cgroup rw,relatime,blkio 0 0\n"; + assertTrue("Directory should be created", blkioCgroup.mkdirs()); + + File mockMtab = new File(parentDir, UUID.randomUUID().toString()); + if (!mockMtab.exists()) { + if (!mockMtab.createNewFile()) { + String message = "Could not create file " + mockMtab.getAbsolutePath(); + throw new IOException(message); + } + } + FileWriter mtabWriter = new FileWriter(mockMtab.getAbsoluteFile()); + mtabWriter.write(cpuMtabContent); + mtabWriter.write(blkioMtabContent); + mtabWriter.close(); + mockMtab.deleteOnExit(); + return mockMtab; } @Test - public void testMountController() { - CGroupsHandler cGroupsHandler = null; + public void testMountController() throws IOException { + File parentDir = new File(tmpPath); + File cgroup = new File(parentDir, controller.getName()); + assertTrue("cgroup dir should be cerated", cgroup.mkdirs()); //Since we enabled (deferred) cgroup controller mounting, no interactions //should have occurred, with this mock verifyZeroInteractions(privilegedOperationExecutorMock); + File emptyMtab = createEmptyCgroups(); try { - cGroupsHandler = new CGroupsHandlerImpl(conf, - privilegedOperationExecutorMock); + CGroupsHandler cGroupsHandler = new CGroupsHandlerImpl(createMountConfiguration(), + privilegedOperationExecutorMock, emptyMtab.getAbsolutePath()); PrivilegedOperation expectedOp = new PrivilegedOperation( PrivilegedOperation.OperationType.MOUNT_CGROUPS); //This is expected to be of the form : //net_cls=/net_cls - StringBuffer controllerKV = new StringBuffer(controller.getName()) - .append('=').append(tmpPath).append('/').append(controller.getName()); - expectedOp.appendArgs(hierarchy, controllerKV.toString()); + String controllerKV = controller.getName() + "=" + tmpPath + + Path.SEPARATOR + controller.getName(); + expectedOp.appendArgs(hierarchy, controllerKV); cGroupsHandler.initializeCGroupController(controller); try { @@ -114,79 +215,90 @@ verifyNoMoreInteractions(privilegedOperationExecutorMock); } catch (PrivilegedOperationException e) { LOG.error("Caught exception: " + e); - Assert.assertTrue("Unexpected PrivilegedOperationException from mock!", + assertTrue("Unexpected PrivilegedOperationException from mock!", false); } } catch (ResourceHandlerException e) { LOG.error("Caught exception: " + e); - Assert.assertTrue("Unexpected ResourceHandler Exception!", false); + assertTrue("Unexpected ResourceHandler Exception!", false); } } @Test - public void testCGroupPaths() { + public void testCGroupPaths() throws IOException { //As per junit behavior, we expect a new mock object to be available //in this test. verifyZeroInteractions(privilegedOperationExecutorMock); CGroupsHandler cGroupsHandler = null; + File mtab = createEmptyCgroups(); + + // Lets manually create a path to (partially) simulate a controller mounted + // later in the test. This is required because the handler uses a mocked + // privileged operation executor + assertTrue("Sample subsystem should be created", + new File(controllerPath).mkdirs()); + try { - cGroupsHandler = new CGroupsHandlerImpl(conf, - privilegedOperationExecutorMock); + cGroupsHandler = new CGroupsHandlerImpl(createMountConfiguration(), + privilegedOperationExecutorMock, mtab.getAbsolutePath()); cGroupsHandler.initializeCGroupController(controller); } catch (ResourceHandlerException e) { LOG.error("Caught exception: " + e); - Assert.assertTrue( + assertTrue( "Unexpected ResourceHandlerException when mounting controller!", false); } String testCGroup = "container_01"; - String expectedPath = new StringBuffer(controllerPath).append('/') - .append(testCGroup).toString(); + String expectedPath = + controllerPath + Path.SEPARATOR + testCGroup; String path = cGroupsHandler.getPathForCGroup(controller, testCGroup); Assert.assertEquals(expectedPath, path); - String expectedPathTasks = new StringBuffer(expectedPath).append('/') - .append(CGroupsHandler.CGROUP_FILE_TASKS).toString(); + String expectedPathTasks = expectedPath + Path.SEPARATOR + + CGroupsHandler.CGROUP_FILE_TASKS; path = cGroupsHandler.getPathForCGroupTasks(controller, testCGroup); Assert.assertEquals(expectedPathTasks, path); String param = CGroupsHandler.CGROUP_PARAM_CLASSID; - String expectedPathParam = new StringBuffer(expectedPath).append('/') - .append(controller.getName()).append('.').append(param).toString(); + String expectedPathParam = expectedPath + Path.SEPARATOR + + controller.getName() + "." + param; path = cGroupsHandler.getPathForCGroupParam(controller, testCGroup, param); Assert.assertEquals(expectedPathParam, path); } @Test - public void testCGroupOperations() { + public void testCGroupOperations() throws IOException { //As per junit behavior, we expect a new mock object to be available //in this test. verifyZeroInteractions(privilegedOperationExecutorMock); CGroupsHandler cGroupsHandler = null; + File mtab = createEmptyCgroups(); + + // Lets manually create a path to (partially) simulate a controller mounted + // later in the test. This is required because the handler uses a mocked + // privileged operation executor + assertTrue("Sample subsystem should be created", + new File(controllerPath).mkdirs()); try { - cGroupsHandler = new CGroupsHandlerImpl(conf, - privilegedOperationExecutorMock); + cGroupsHandler = new CGroupsHandlerImpl(createMountConfiguration(), + privilegedOperationExecutorMock, mtab.getAbsolutePath()); cGroupsHandler.initializeCGroupController(controller); } catch (ResourceHandlerException e) { LOG.error("Caught exception: " + e); - Assert.assertTrue( + assertTrue( "Unexpected ResourceHandlerException when mounting controller!", false); } - //Lets manually create a path to (partially) simulate a mounted controller - //this is required because the handler uses a mocked privileged operation - //executor - new File(controllerPath).mkdirs(); String testCGroup = "container_01"; - String expectedPath = new StringBuffer(controllerPath).append('/') - .append(testCGroup).toString(); + String expectedPath = controllerPath + + Path.SEPARATOR + testCGroup; try { String path = cGroupsHandler.createCGroup(controller, testCGroup); - Assert.assertTrue(new File(expectedPath).exists()); + assertTrue(new File(expectedPath).exists()); Assert.assertEquals(expectedPath, path); //update param and read param tests. @@ -199,11 +311,12 @@ cGroupsHandler .updateCGroupParam(controller, testCGroup, param, paramValue); - String paramPath = new StringBuffer(expectedPath).append('/') - .append(controller.getName()).append('.').append(param).toString(); + String paramPath = expectedPath + + Path.SEPARATOR + controller.getName() + + "." + param; File paramFile = new File(paramPath); - Assert.assertTrue(paramFile.exists()); + assertTrue(paramFile.exists()); try { Assert.assertEquals(paramValue, new String(Files.readAllBytes( paramFile.toPath()))); @@ -230,47 +343,6 @@ } } - public static File createMockCgroupMount(File parentDir, String type) - throws IOException { - return createMockCgroupMount(parentDir, type, "hadoop-yarn"); - } - - private static File createMockCgroupMount(File parentDir, String type, - String hierarchy) throws IOException { - File cgroupMountDir = - new File(parentDir.getAbsolutePath(), type + "/" + hierarchy); - FileUtils.deleteQuietly(cgroupMountDir); - if (!cgroupMountDir.mkdirs()) { - String message = - "Could not create dir " + cgroupMountDir.getAbsolutePath(); - throw new IOException(message); - } - return cgroupMountDir; - } - - public static File createMockMTab(File parentDir) throws IOException { - String cpuMtabContent = - "none " + parentDir.getAbsolutePath() - + "/cpu cgroup rw,relatime,cpu 0 0\n"; - String blkioMtabContent = - "none " + parentDir.getAbsolutePath() - + "/blkio cgroup rw,relatime,blkio 0 0\n"; - - File mockMtab = new File(parentDir, UUID.randomUUID().toString()); - if (!mockMtab.exists()) { - if (!mockMtab.createNewFile()) { - String message = "Could not create file " + mockMtab.getAbsolutePath(); - throw new IOException(message); - } - } - FileWriter mtabWriter = new FileWriter(mockMtab.getAbsoluteFile()); - mtabWriter.write(cpuMtabContent); - mtabWriter.write(blkioMtabContent); - mtabWriter.close(); - mockMtab.deleteOnExit(); - return mockMtab; - } - /** * Tests whether mtab parsing works as expected with a valid hierarchy set. * @throws Exception the test will fail @@ -280,24 +352,20 @@ // Initialize mtab and cgroup dir File parentDir = new File(tmpPath); // create mock cgroup - File cpuCgroupMountDir = createMockCgroupMount(parentDir, "cpu", - hierarchy); - Assert.assertTrue(cpuCgroupMountDir.exists()); - File blkioCgroupMountDir = createMockCgroupMount(parentDir, - "blkio", hierarchy); - Assert.assertTrue(blkioCgroupMountDir.exists()); - File mockMtabFile = createMockMTab(parentDir); + File mockMtabFile = createPremountedCgroups(parentDir, false); // Run mtabs parsing + Map> newMtab = + CGroupsHandlerImpl.parseMtab(mockMtabFile.getAbsolutePath()); Map controllerPaths = CGroupsHandlerImpl.initializeControllerPathsFromMtab( - mockMtabFile.getAbsolutePath(), hierarchy); + newMtab); // Verify Assert.assertEquals(2, controllerPaths.size()); - Assert.assertTrue(controllerPaths + assertTrue(controllerPaths .containsKey(CGroupsHandler.CGroupController.CPU)); - Assert.assertTrue(controllerPaths + assertTrue(controllerPaths .containsKey(CGroupsHandler.CGroupController.BLKIO)); String cpuDir = controllerPaths.get(CGroupsHandler.CGroupController.CPU); String blkioDir = @@ -306,17 +374,6 @@ Assert.assertEquals(parentDir.getAbsolutePath() + "/blkio", blkioDir); } - /** - * Tests whether mtab parsing works as expected with an empty hierarchy set. - * @throws Exception the test will fail - */ - @Test - public void testPreMountedController() throws Exception { - testPreMountedControllerInitialization("hadoop-yarn"); - testPreMountedControllerInitialization(""); - testPreMountedControllerInitialization("/"); - } - /** * Tests whether mtab parsing works as expected with the specified hierarchy. * @param myHierarchy path to local cgroup hierarchy @@ -326,64 +383,56 @@ throws Exception { // Initialize mount point File parentDir = new File(tmpPath); - FileUtils.deleteQuietly(parentDir); - Assert.assertTrue("Could not create dirs", parentDir.mkdirs()); - File mtab = createMockMTab(parentDir); + File mtab = createPremountedCgroups(parentDir, false); File mountPoint = new File(parentDir, "cpu"); - File cpuCgroupMountDir = createMockCgroupMount( - parentDir, "cpu", myHierarchy); // Initialize Yarn classes - Configuration confNoMount = new Configuration(); - confNoMount.set(YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_HIERARCHY, - myHierarchy); - confNoMount.setBoolean(YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_MOUNT, - false); + Configuration confNoMount = createNoMountConfiguration(myHierarchy); CGroupsHandlerImpl cGroupsHandler = new CGroupsHandlerImpl(confNoMount, privilegedOperationExecutorMock, mtab.getAbsolutePath()); - + File cpuCgroupMountDir = new File( + cGroupsHandler.getPathForCGroup(CGroupsHandler.CGroupController.CPU, "")); // Test that a missing yarn hierarchy will be created automatically if (!cpuCgroupMountDir.equals(mountPoint)) { - Assert.assertTrue("Could not delete cgroups", cpuCgroupMountDir.delete()); - Assert.assertTrue("Directory should be deleted", + assertTrue("Directory should be deleted", !cpuCgroupMountDir.exists()); } cGroupsHandler.initializeCGroupController( CGroupsHandler.CGroupController.CPU); - Assert.assertTrue("Cgroups not writable", cpuCgroupMountDir.exists() && + assertTrue("Cgroups not writable", cpuCgroupMountDir.exists() && cpuCgroupMountDir.canWrite()); // Test that an inaccessible yarn hierarchy results in an exception - Assert.assertTrue(cpuCgroupMountDir.setWritable(false)); + assertTrue(cpuCgroupMountDir.setWritable(false)); try { cGroupsHandler.initializeCGroupController( CGroupsHandler.CGroupController.CPU); Assert.fail("An inaccessible path should result in an exception"); } catch (Exception e) { - Assert.assertTrue("Unexpected exception " + e.getClass().toString(), + assertTrue("Unexpected exception " + e.getClass().toString(), e instanceof ResourceHandlerException); } finally { - Assert.assertTrue("Could not revert writable permission", + assertTrue("Could not revert writable permission", cpuCgroupMountDir.setWritable(true)); } // Test that a non-accessible mount directory results in an exception if (!cpuCgroupMountDir.equals(mountPoint)) { - Assert.assertTrue("Could not delete cgroups", cpuCgroupMountDir.delete()); - Assert.assertTrue("Directory should be deleted", + assertTrue("Could not delete cgroups", cpuCgroupMountDir.delete()); + assertTrue("Directory should be deleted", !cpuCgroupMountDir.exists()); } - Assert.assertTrue(mountPoint.setWritable(false)); + assertTrue(mountPoint.setWritable(false)); try { cGroupsHandler.initializeCGroupController( CGroupsHandler.CGroupController.CPU); Assert.fail("An inaccessible path should result in an exception"); } catch (Exception e) { - Assert.assertTrue("Unexpected exception " + e.getClass().toString(), + assertTrue("Unexpected exception " + e.getClass().toString(), e instanceof ResourceHandlerException); } finally { - Assert.assertTrue("Could not revert writable permission", + assertTrue("Could not revert writable permission", mountPoint.setWritable(true)); } @@ -391,7 +440,7 @@ if (!cpuCgroupMountDir.equals(mountPoint)) { Assert.assertFalse("Could not delete cgroups", cpuCgroupMountDir.delete()); - Assert.assertTrue("Directory should be deleted", + assertTrue("Directory should be deleted", !cpuCgroupMountDir.exists()); SecurityManager manager = System.getSecurityManager(); System.setSecurityManager(new MockSecurityManagerDenyWrite()); @@ -400,7 +449,7 @@ CGroupsHandler.CGroupController.CPU); Assert.fail("An inaccessible path should result in an exception"); } catch (Exception e) { - Assert.assertTrue("Unexpected exception " + e.getClass().toString(), + assertTrue("Unexpected exception " + e.getClass().toString(), e instanceof ResourceHandlerException); } finally { System.setSecurityManager(manager); @@ -411,33 +460,85 @@ if (!cpuCgroupMountDir.equals(mountPoint)) { Assert.assertFalse("Could not delete cgroups", cpuCgroupMountDir.delete()); - Assert.assertTrue("Directory should be deleted", + assertTrue("Directory should be deleted", !cpuCgroupMountDir.exists()); } FileUtils.deleteQuietly(mountPoint); - Assert.assertTrue("cgroups mount point should be deleted", + assertTrue("cgroups mount point should be deleted", !mountPoint.exists()); try { cGroupsHandler.initializeCGroupController( CGroupsHandler.CGroupController.CPU); Assert.fail("An inaccessible path should result in an exception"); } catch (Exception e) { - Assert.assertTrue("Unexpected exception " + e.getClass().toString(), + assertTrue("Unexpected exception " + e.getClass().toString(), e instanceof ResourceHandlerException); } } - @After - public void teardown() { - FileUtil.fullyDelete(new File(tmpPath)); + /** + * Tests whether mtab parsing works as expected with an normal hierarchy set. + * @throws Exception the test will fail + */ + @Test + public void testPreMountedController() throws Exception { + testPreMountedControllerInitialization(hierarchy); } - private class MockSecurityManagerDenyWrite extends SecurityManager { - @Override - public void checkPermission(Permission perm) { - if(perm.getActions().equals("write")) { - throw new SecurityException("Mock not allowed"); - } - } + /** + * Tests whether mtab parsing works as expected with an empty hierarchy set. + * @throws Exception the test will fail + */ + @Test + public void testPreMountedControllerEmpty() throws Exception { + testPreMountedControllerInitialization(""); + } + + /** + * Tests whether mtab parsing works as expected with a / hierarchy set. + * @throws Exception the test will fail + */ + @Test + public void testPreMountedControllerRoot() throws Exception { + testPreMountedControllerInitialization("/"); + } + + /** + * Tests whether mtab parsing works as expected with the specified hierarchy. + * @throws Exception the test will fail + */ + @Test + public void testRemount() + throws Exception { + // Initialize mount point + File parentDir = new File(tmpPath); + + final String oldMountPointDir = "oldmount"; + final String newMountPointDir = "newmount"; + + File oldMountPoint = new File(parentDir, oldMountPointDir); + File mtab = createPremountedCgroups( + oldMountPoint, true); + + File newMountPoint = new File(parentDir, newMountPointDir); + assertTrue("Could not create dirs", new File(newMountPoint, "cpu").mkdirs()); + + // Initialize Yarn classes + Configuration confMount = createMountConfiguration(); + confMount.set(YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_MOUNT_PATH, + parentDir.getAbsolutePath() + Path.SEPARATOR + newMountPointDir); + CGroupsHandlerImpl cGroupsHandler = new CGroupsHandlerImpl(confMount, + privilegedOperationExecutorMock, mtab.getAbsolutePath()); + + cGroupsHandler.initializeCGroupController( + CGroupsHandler.CGroupController.CPU); + + ArgumentCaptor opCaptor = ArgumentCaptor.forClass( + PrivilegedOperation.class); + verify(privilegedOperationExecutorMock) + .executePrivilegedOperation(opCaptor.capture(), eq(false)); + File hierarchy = + new File(new File(newMountPoint, "cpu"), this.hierarchy); + assertTrue("Yarn cgroup should exist", hierarchy.exists()); } } \ No newline at end of file Index: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandler.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandler.java (revision be4ac51995e02e9bfefa3778e57c60c9658ea741) +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandler.java (revision 8b96314eeca04073bbac728cba6196722f2faead) @@ -39,7 +39,11 @@ CPU("cpu"), NET_CLS("net_cls"), BLKIO("blkio"), - MEMORY("memory"); + MEMORY("memory"), + CPUACCT("cpuacct"), + CPUSET("cpuset"), + FREEZER("freezer"), + DEVICES("devices"); private final String name; Index: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestCgroupsLCEResourcesHandler.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestCgroupsLCEResourcesHandler.java (revision b91bd2fc8a45df40924a21d08ac08202b49a1fb3) +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestCgroupsLCEResourcesHandler.java (revision d15d54ef93b59f31c5e022cb38484f7df3ff2bdf) @@ -151,6 +151,24 @@ } } + public static File createMockCgroupMount(File parentDir, String type) + throws IOException { + return createMockCgroupMount(parentDir, type, "hadoop-yarn"); + } + + private static File createMockCgroupMount(File parentDir, String type, + String hierarchy) throws IOException { + File cgroupMountDir = + new File(parentDir.getAbsolutePath(), type + "/" + hierarchy); + FileUtils.deleteQuietly(cgroupMountDir); + if (!cgroupMountDir.mkdirs()) { + String message = + "Could not create dir " + cgroupMountDir.getAbsolutePath(); + throw new IOException(message); + } + return cgroupMountDir; + } + @Test public void testInit() throws IOException { LinuxContainerExecutor mockLCE = new MockLinuxContainerExecutor(); @@ -165,13 +183,14 @@ handler.setConf(conf); handler.initConfig(); + // create mock mtab + File mockMtab = + TestCGroupsHandlerImpl.createPremountedCgroups(cgroupDir, false); + // create mock cgroup - File cpuCgroupMountDir = TestCGroupsHandlerImpl.createMockCgroupMount( + File cpuCgroupMountDir = createMockCgroupMount( cgroupDir, "cpu"); - // create mock mtab - File mockMtab = TestCGroupsHandlerImpl.createMockMTab(cgroupDir); - // setup our handler and call init() handler.setMtabFile(mockMtab.getAbsolutePath()); @@ -262,13 +281,14 @@ handler.setConf(conf); handler.initConfig(); + // create mock mtab + File mockMtab = + TestCGroupsHandlerImpl.createPremountedCgroups(cgroupDir, false); + // create mock cgroup - File cpuCgroupMountDir = TestCGroupsHandlerImpl.createMockCgroupMount( + File cpuCgroupMountDir = createMockCgroupMount( cgroupDir, "cpu"); - // create mock mtab - File mockMtab = TestCGroupsHandlerImpl.createMockMTab(cgroupDir); - // setup our handler and call init() handler.setMtabFile(mockMtab.getAbsolutePath()); handler.init(mockLCE, plugin);