diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 81c9cb2..96e9c19 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -1517,11 +1517,21 @@ - Where the LCE should attempt to mount cgroups if not found. Common locations - include /sys/fs/cgroup and /cgroup; the default location can vary depending on the Linux - distribution in use. This path must exist before the NodeManager is launched. - Only used when the LCE resources handler is set to the CgroupsLCEResourcesHandler, and - yarn.nodemanager.linux-container-executor.cgroups.mount is true. + Requested cgroup mount path. Yarn has built in functionality to discover + the system cgroup mount paths, so use this setting only, if the discovery does not work. + + This path must exist before the NodeManager is launched. + The location can vary depending on the Linux distribution in use. + Common locations include /sys/fs/cgroup and /cgroup. + + If cgroups are not mounted, set yarn.nodemanager.linux-container-executor.cgroups.mount + to true. In this case it specifies, where the LCE should attempt to mount cgroups if not found. + + If cgroups is accessible through lxcfs or some other file systems, + then set this path and yarn.nodemanager.linux-container-executor.cgroups.mount to false. + Yarn tries to use this path first, before any cgroup mount point discovery. + Only used when the LCE resources handler is set to the CgroupsLCEResourcesHandler + yarn.nodemanager.linux-container-executor.cgroups.mount-path diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandler.java index 8fc35a8..82bd366 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandler.java @@ -23,6 +23,9 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +import java.util.HashSet; +import java.util.Set; + /** * Provides CGroups functionality. Implementations are expected to be * thread-safe @@ -54,6 +57,18 @@ String getName() { return name; } + + /** + * Get the list of valid cgroup names. + * @return The set of cgroup name strings + */ + public static Set getValidCGroups() { + HashSet validCgroups = new HashSet<>(); + for (CGroupController controller : CGroupController.values()) { + validCgroups.add(controller.getName()); + } + return validCgroups; + } } String CGROUP_FILE_TASKS = "tasks"; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandlerImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandlerImpl.java index 85b01cd..77bb071 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandlerImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandlerImpl.java @@ -142,11 +142,18 @@ private void initializeControllerPaths() throws ResourceHandlerException { // the same hierarchy will be mounted at each mount point with the same // subsystem set. - Map> newMtab; + Map> newMtab = null; Map cPaths; try { - // parse mtab - newMtab = parseMtab(mtabFile); + if (this.cGroupMountPath != null && !this.enableCGroupMount) { + newMtab = ResourceHandlerModule. + parseConfiguredCGroupPath(this.cGroupMountPath); + } + + if (newMtab == null) { + // parse mtab + newMtab = parseMtab(mtabFile); + } // find cgroup controller paths cPaths = initializeControllerPathsFromMtab(newMtab); @@ -203,10 +210,8 @@ private void initializeControllerPaths() throws ResourceHandlerException { throws IOException { Map> ret = new HashMap<>(); BufferedReader in = null; - HashSet validCgroups = new HashSet<>(); - for (CGroupController controller : CGroupController.values()) { - validCgroups.add(controller.getName()); - } + Set validCgroups = + CGroupsHandler.CGroupController.getValidCGroups(); try { FileInputStream fis = new FileInputStream(new File(mtab)); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/ResourceHandlerModule.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/ResourceHandlerModule.java index 7fc04bd..9e83c27 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/ResourceHandlerModule.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/ResourceHandlerModule.java @@ -31,6 +31,13 @@ import org.apache.hadoop.yarn.server.nodemanager.util.CgroupsLCEResourcesHandler; import org.apache.hadoop.yarn.server.nodemanager.util.DefaultLCEResourcesHandler; +import java.io.File; +import java.io.IOException; +import java.util.Set; +import java.util.HashSet; +import java.util.Map; +import java.util.HashMap; +import java.util.Arrays; import java.util.ArrayList; import java.util.List; @@ -229,4 +236,45 @@ public static ResourceHandlerChain getConfiguredResourceHandlerChain( static void nullifyResourceHandlerChain() throws ResourceHandlerException { resourceHandlerChain = null; } + + /** + * If a cgroup mount directory is specified, it returns cgroup directories + * with valid names. + * The requirement is that each hierarchy has to be named with the comma + * separated names of subsystems supported. + * For example: /sys/fs/cgroup/cpu,cpuacct + * @param cgroupMountPath Root cgroup mount path (/sys/fs/cgroup in the + * example above) + * @return A path to cgroup subsystem set mapping in the same format as + * {@link CGroupsHandlerImpl#parseMtab(String)} + * @throws IOException if the specified directory cannot be listed + */ + public static Map> parseConfiguredCGroupPath( + String cgroupMountPath) throws IOException { + File cgroupDir = new File(cgroupMountPath); + File[] list = cgroupDir.listFiles(); + if (list == null) { + throw new IOException("Empty cgroup mount directory specified: " + + cgroupMountPath); + } + + Map> pathSubsystemMappings = new HashMap<>(); + Set validCGroups = + CGroupsHandler.CGroupController.getValidCGroups(); + for (File candidate: list) { + Set cgroupList = + new HashSet<>(Arrays.asList(candidate.getName().split(","))); + // Collect the valid subsystem names + cgroupList.retainAll(validCGroups); + if (!cgroupList.isEmpty()) { + if (candidate.isDirectory() && candidate.canWrite()) { + pathSubsystemMappings.put(candidate.getAbsolutePath(), cgroupList); + } else { + LOG.warn("The following cgroup is not a directory or it is not" + + " writable" + candidate.getAbsolutePath()); + } + } + } + return pathSubsystemMappings; + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/CgroupsLCEResourcesHandler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/CgroupsLCEResourcesHandler.java index bca4fdc..48de453 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/CgroupsLCEResourcesHandler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/CgroupsLCEResourcesHandler.java @@ -27,6 +27,7 @@ import java.io.OutputStreamWriter; import java.io.PrintWriter; import java.io.Writer; +import java.util.Arrays; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -39,7 +40,6 @@ import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.Sets; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; @@ -51,6 +51,8 @@ import org.apache.hadoop.yarn.server.nodemanager.LinuxContainerExecutor; import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperation; import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.CGroupsCpuResourceHandlerImpl; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.CGroupsHandler; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerModule; import org.apache.hadoop.yarn.util.Clock; import org.apache.hadoop.yarn.util.ResourceCalculatorPlugin; import org.apache.hadoop.yarn.util.SystemClock; @@ -400,6 +402,8 @@ public String getResourcesOption(ContainerId containerId) { private Map> parseMtab() throws IOException { Map> ret = new HashMap>(); BufferedReader in = null; + Set validCgroups = + CGroupsHandler.CGroupController.getValidCGroups(); try { FileInputStream fis = new FileInputStream(new File(getMtabFileName())); @@ -415,8 +419,11 @@ public String getResourcesOption(ContainerId containerId) { String options = m.group(3); if (type.equals(CGROUPS_FSTYPE)) { - HashSet value = Sets.newHashSet(options.split(",")); - ret.put(path, value); + Set cgroupList = + new HashSet<>(Arrays.asList(options.split(","))); + // Collect the valid subsystem names + cgroupList.retainAll(validCgroups); + ret.put(path, cgroupList); } } } @@ -448,7 +455,16 @@ String findControllerInMtab(String controller, private void initializeControllerPaths() throws IOException { String controllerPath; - Map> parsedMtab = parseMtab(); + Map> parsedMtab = null; + + if (this.cgroupMountPath != null && !this.cgroupMount) { + parsedMtab = ResourceHandlerModule. + parseConfiguredCGroupPath(this.cgroupMountPath); + } + + if (parsedMtab == null) { + parsedMtab = parseMtab(); + } // CPU diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsHandlerImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsHandlerImpl.java index dd8e338..e678db0 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsHandlerImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsHandlerImpl.java @@ -573,4 +573,29 @@ public void testRemount() new File(new File(newMountPoint, "cpu"), this.hierarchy); assertTrue("Yarn cgroup should exist", hierarchyFile.exists()); } + + + @Test + public void testManualCgroupSetting() throws ResourceHandlerException { + YarnConfiguration conf = new YarnConfiguration(); + conf.set(YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_MOUNT_PATH, tmpPath); + conf.set(YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_HIERARCHY, + "/hadoop-yarn"); + File cpu = new File(new File(tmpPath, "cpuacct,cpu"), "/hadoop-yarn"); + + try { + Assert.assertTrue("temp dir should be created", cpu.mkdirs()); + + CGroupsHandlerImpl cGroupsHandler = new CGroupsHandlerImpl(conf, null); + cGroupsHandler.initializeCGroupController( + CGroupsHandler.CGroupController.CPU); + + Assert.assertEquals(cpu.getAbsolutePath(), + new File(cGroupsHandler.getPathForCGroup( + CGroupsHandler.CGroupController.CPU, "")).getAbsolutePath()); + + } finally { + FileUtils.deleteQuietly(cpu); + } + } } \ No newline at end of file diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestCgroupsLCEResourcesHandler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestCgroupsLCEResourcesHandler.java index b562133..7aa061d 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestCgroupsLCEResourcesHandler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestCgroupsLCEResourcesHandler.java @@ -41,6 +41,8 @@ import java.util.Set; import java.util.concurrent.CountDownLatch; +import static org.mockito.Mockito.when; + @Deprecated public class TestCgroupsLCEResourcesHandler { private static File cgroupDir = null; @@ -388,4 +390,33 @@ public void testSelectCgroup() { FileUtils.deleteQuietly(memory); } } + + @Test + public void testManualCgroupSetting() throws IOException { + CgroupsLCEResourcesHandler handler = new CgroupsLCEResourcesHandler(); + YarnConfiguration conf = new YarnConfiguration(); + conf.set(YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_MOUNT_PATH, + cgroupDir.getAbsolutePath()); + handler.setConf(conf); + File cpu = new File(new File(cgroupDir, "cpuacct,cpu"), "/hadoop-yarn"); + + try { + Assert.assertTrue("temp dir should be created", cpu.mkdirs()); + + final int numProcessors = 4; + ResourceCalculatorPlugin plugin = + Mockito.mock(ResourceCalculatorPlugin.class); + Mockito.doReturn(numProcessors).when(plugin).getNumProcessors(); + Mockito.doReturn(numProcessors).when(plugin).getNumCores(); + when(plugin.getNumProcessors()).thenReturn(8); + handler.init(null, plugin); + + Assert.assertEquals(cpu.getParent(), + handler.getControllerPaths().get("cpu")); + + } finally { + FileUtils.deleteQuietly(cpu); + } + } + }