Index: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml (date 1501180845000) +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml (date 1501266963000) @@ -1521,17 +1521,25 @@ Whether the LCE should attempt to mount cgroups if not found. - Only used when the LCE resources handler is set to the CgroupsLCEResourcesHandler. + Only used when the LCE resources handler is set to the + CgroupsLCEResourcesHandler. + yarn.nodemanager.linux-container-executor.cgroups.mount false - Where the LCE should attempt to mount cgroups if not found. Common locations - include /sys/fs/cgroup and /cgroup; the default location can vary depending on the Linux - distribution in use. This path must exist before the NodeManager is launched. - Only used when the LCE resources handler is set to the CgroupsLCEResourcesHandler, and - yarn.nodemanager.linux-container-executor.cgroups.mount is true. + Requested CGroup mount path. Yarn has built in functionality to + discover the system cgroup mount paths, so use this setting only, if the + discovery does not work. + This path must exist before the NodeManager is launched. The usage of this + path depends on yarn.nodemanager.linux-container-executor.cgroups.mount. + If it is true, Yarn tries to mount CGroups here. + If it is false Yarn tries to use CGroups from here. + Please refer to NodeManagerCgroups.html in the documentation for details. + Only used when the LCE resources handler is set to the + CgroupsLCEResourcesHandler. + yarn.nodemanager.linux-container-executor.cgroups.mount-path Index: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandler.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandler.java (date 1501180845000) +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandler.java (date 1501266963000) @@ -23,6 +23,9 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +import java.util.HashSet; +import java.util.Set; + /** * Provides CGroups functionality. Implementations are expected to be * thread-safe @@ -54,6 +57,18 @@ String getName() { return name; } + + /** + * Get the list of valid cgroup names. + * @return The set of cgroup name strings + */ + public static Set getValidCGroups() { + HashSet validCgroups = new HashSet<>(); + for (CGroupController controller : CGroupController.values()) { + validCgroups.add(controller.getName()); + } + return validCgroups; + } } String CGROUP_FILE_TASKS = "tasks"; Index: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandlerImpl.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandlerImpl.java (date 1501180845000) +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandlerImpl.java (date 1501266963000) @@ -83,7 +83,7 @@ * @param mtab mount file location * @throws ResourceHandlerException if initialization failed */ - public CGroupsHandlerImpl(Configuration conf, PrivilegedOperationExecutor + CGroupsHandlerImpl(Configuration conf, PrivilegedOperationExecutor privilegedOperationExecutor, String mtab) throws ResourceHandlerException { this.cGroupPrefix = conf.get(YarnConfiguration. @@ -115,7 +115,7 @@ * PrivilegedContainerOperations * @throws ResourceHandlerException if initialization failed */ - public CGroupsHandlerImpl(Configuration conf, PrivilegedOperationExecutor + CGroupsHandlerImpl(Configuration conf, PrivilegedOperationExecutor privilegedOperationExecutor) throws ResourceHandlerException { this(conf, privilegedOperationExecutor, MTAB_FILE); } @@ -142,11 +142,18 @@ // the same hierarchy will be mounted at each mount point with the same // subsystem set. - Map> newMtab; + Map> newMtab = null; Map cPaths; try { - // parse mtab - newMtab = parseMtab(mtabFile); + if (this.cGroupMountPath != null && !this.enableCGroupMount) { + newMtab = ResourceHandlerModule. + parseConfiguredCGroupPath(this.cGroupMountPath); + } + + if (newMtab == null) { + // parse mtab + newMtab = parseMtab(mtabFile); + } // find cgroup controller paths cPaths = initializeControllerPathsFromMtab(newMtab); @@ -203,10 +210,8 @@ throws IOException { Map> ret = new HashMap<>(); BufferedReader in = null; - HashSet validCgroups = new HashSet<>(); - for (CGroupController controller : CGroupController.values()) { - validCgroups.add(controller.getName()); - } + Set validCgroups = + CGroupsHandler.CGroupController.getValidCGroups(); try { FileInputStream fis = new FileInputStream(new File(mtab)); @@ -487,7 +492,8 @@ try (BufferedReader inl = new BufferedReader(new InputStreamReader(new FileInputStream(cgf + "/tasks"), "UTF-8"))) { - if ((str = inl.readLine()) != null) { + str = inl.readLine(); + if (str != null) { LOG.debug("First line in cgroup tasks file: " + cgf + " " + str); } } catch (IOException e) { Index: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/ResourceHandlerModule.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/ResourceHandlerModule.java (date 1501180845000) +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/ResourceHandlerModule.java (date 1501266963000) @@ -31,6 +31,13 @@ import org.apache.hadoop.yarn.server.nodemanager.util.CgroupsLCEResourcesHandler; import org.apache.hadoop.yarn.server.nodemanager.util.DefaultLCEResourcesHandler; +import java.io.File; +import java.io.IOException; +import java.util.Set; +import java.util.HashSet; +import java.util.Map; +import java.util.HashMap; +import java.util.Arrays; import java.util.ArrayList; import java.util.List; @@ -113,7 +120,7 @@ } private static TrafficControlBandwidthHandlerImpl - getTrafficControlBandwidthHandler(Configuration conf) + getTrafficControlBandwidthHandler(Configuration conf) throws ResourceHandlerException { if (conf.getBoolean(YarnConfiguration.NM_NETWORK_RESOURCE_ENABLED, YarnConfiguration.DEFAULT_NM_NETWORK_RESOURCE_ENABLED)) { @@ -137,7 +144,7 @@ } public static OutboundBandwidthResourceHandler - getOutboundBandwidthResourceHandler(Configuration conf) + getOutboundBandwidthResourceHandler(Configuration conf) throws ResourceHandlerException { return getTrafficControlBandwidthHandler(conf); } @@ -176,7 +183,7 @@ } private static CGroupsMemoryResourceHandlerImpl - getCgroupsMemoryResourceHandler( + getCgroupsMemoryResourceHandler( Configuration conf) throws ResourceHandlerException { if (cGroupsMemoryResourceHandler == null) { synchronized (MemoryResourceHandler.class) { @@ -229,4 +236,45 @@ static void nullifyResourceHandlerChain() throws ResourceHandlerException { resourceHandlerChain = null; } + + /** + * If a cgroup mount directory is specified, it returns cgroup directories + * with valid names. + * The requirement is that each hierarchy has to be named with the comma + * separated names of subsystems supported. + * For example: /sys/fs/cgroup/cpu,cpuacct + * @param cgroupMountPath Root cgroup mount path (/sys/fs/cgroup in the + * example above) + * @return A path to cgroup subsystem set mapping in the same format as + * {@link CGroupsHandlerImpl#parseMtab(String)} + * @throws IOException if the specified directory cannot be listed + */ + public static Map> parseConfiguredCGroupPath( + String cgroupMountPath) throws IOException { + File cgroupDir = new File(cgroupMountPath); + File[] list = cgroupDir.listFiles(); + if (list == null) { + throw new IOException("Empty cgroup mount directory specified: " + + cgroupMountPath); + } + + Map> pathSubsystemMappings = new HashMap<>(); + Set validCGroups = + CGroupsHandler.CGroupController.getValidCGroups(); + for (File candidate: list) { + Set cgroupList = + new HashSet<>(Arrays.asList(candidate.getName().split(","))); + // Collect the valid subsystem names + cgroupList.retainAll(validCGroups); + if (!cgroupList.isEmpty()) { + if (candidate.isDirectory() && candidate.canWrite()) { + pathSubsystemMappings.put(candidate.getAbsolutePath(), cgroupList); + } else { + LOG.warn("The following cgroup is not a directory or it is not" + + " writable" + candidate.getAbsolutePath()); + } + } + } + return pathSubsystemMappings; + } } Index: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/CgroupsLCEResourcesHandler.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/CgroupsLCEResourcesHandler.java (date 1501180845000) +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/CgroupsLCEResourcesHandler.java (date 1501266963000) @@ -27,6 +27,7 @@ import java.io.OutputStreamWriter; import java.io.PrintWriter; import java.io.Writer; +import java.util.Arrays; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -39,7 +40,6 @@ import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.Sets; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; @@ -51,6 +51,8 @@ import org.apache.hadoop.yarn.server.nodemanager.LinuxContainerExecutor; import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperation; import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.CGroupsCpuResourceHandlerImpl; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.CGroupsHandler; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerModule; import org.apache.hadoop.yarn.util.Clock; import org.apache.hadoop.yarn.util.ResourceCalculatorPlugin; import org.apache.hadoop.yarn.util.SystemClock; @@ -87,11 +89,11 @@ private long deleteCgroupTimeout; private long deleteCgroupDelay; - // package private for testing purposes + @VisibleForTesting Clock clock; private float yarnProcessors; - int nodeVCores; + private int nodeVCores; public CgroupsLCEResourcesHandler() { this.controllerPaths = new HashMap(); @@ -132,8 +134,10 @@ this.strictResourceUsageMode = conf .getBoolean( - YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_STRICT_RESOURCE_USAGE, - YarnConfiguration.DEFAULT_NM_LINUX_CONTAINER_CGROUPS_STRICT_RESOURCE_USAGE); + YarnConfiguration + .NM_LINUX_CONTAINER_CGROUPS_STRICT_RESOURCE_USAGE, + YarnConfiguration + .DEFAULT_NM_LINUX_CONTAINER_CGROUPS_STRICT_RESOURCE_USAGE); int len = cgroupPrefix.length(); if (cgroupPrefix.charAt(len - 1) == '/') { @@ -169,8 +173,10 @@ if (systemProcessors != (int) yarnProcessors) { LOG.info("YARN containers restricted to " + yarnProcessors + " cores"); int[] limits = getOverallLimits(yarnProcessors); - updateCgroup(CONTROLLER_CPU, "", CPU_PERIOD_US, String.valueOf(limits[0])); - updateCgroup(CONTROLLER_CPU, "", CPU_QUOTA_US, String.valueOf(limits[1])); + updateCgroup(CONTROLLER_CPU, "", CPU_PERIOD_US, + String.valueOf(limits[0])); + updateCgroup(CONTROLLER_CPU, "", CPU_QUOTA_US, + String.valueOf(limits[1])); } else if (CGroupsCpuResourceHandlerImpl.cpuLimitsExist( pathForCgroup(CONTROLLER_CPU, ""))) { LOG.info("Removing CPU constraints for YARN containers."); @@ -178,8 +184,8 @@ } } - int[] getOverallLimits(float yarnProcessors) { - return CGroupsCpuResourceHandlerImpl.getOverallLimits(yarnProcessors); + int[] getOverallLimits(float yarnProcessorsArg) { + return CGroupsCpuResourceHandlerImpl.getOverallLimits(yarnProcessorsArg); } @@ -204,7 +210,7 @@ LOG.debug("createCgroup: " + path); } - if (! new File(path).mkdir()) { + if (!new File(path).mkdir()) { throw new IOException("Failed to create cgroup at " + path); } } @@ -251,7 +257,8 @@ try (BufferedReader inl = new BufferedReader(new InputStreamReader(new FileInputStream(cgf + "/tasks"), "UTF-8"))) { - if ((str = inl.readLine()) != null) { + str = inl.readLine(); + if (str != null) { LOG.debug("First line in cgroup tasks file: " + cgf + " " + str); } } catch (IOException e) { @@ -337,9 +344,9 @@ (containerVCores * yarnProcessors) / (float) nodeVCores; int[] limits = getOverallLimits(containerCPU); updateCgroup(CONTROLLER_CPU, containerName, CPU_PERIOD_US, - String.valueOf(limits[0])); + String.valueOf(limits[0])); updateCgroup(CONTROLLER_CPU, containerName, CPU_QUOTA_US, - String.valueOf(limits[1])); + String.valueOf(limits[1])); } } } @@ -400,6 +407,8 @@ private Map> parseMtab() throws IOException { Map> ret = new HashMap>(); BufferedReader in = null; + Set validCgroups = + CGroupsHandler.CGroupController.getValidCGroups(); try { FileInputStream fis = new FileInputStream(new File(getMtabFileName())); @@ -415,8 +424,11 @@ String options = m.group(3); if (type.equals(CGROUPS_FSTYPE)) { - HashSet value = Sets.newHashSet(options.split(",")); - ret.put(path, value); + Set cgroupList = + new HashSet<>(Arrays.asList(options.split(","))); + // Collect the valid subsystem names + cgroupList.retainAll(validCgroups); + ret.put(path, cgroupList); } } } @@ -448,7 +460,16 @@ private void initializeControllerPaths() throws IOException { String controllerPath; - Map> parsedMtab = parseMtab(); + Map> parsedMtab = null; + + if (this.cgroupMountPath != null && !this.cgroupMount) { + parsedMtab = ResourceHandlerModule. + parseConfiguredCGroupPath(this.cgroupMountPath); + } + + if (parsedMtab == null) { + parsedMtab = parseMtab(); + } // CPU Index: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsHandlerImpl.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsHandlerImpl.java (date 1501180845000) +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsHandlerImpl.java (date 1501266963000) @@ -573,4 +573,29 @@ new File(new File(newMountPoint, "cpu"), this.hierarchy); assertTrue("Yarn cgroup should exist", hierarchyFile.exists()); } + + + @Test + public void testManualCgroupSetting() throws ResourceHandlerException { + YarnConfiguration conf = new YarnConfiguration(); + conf.set(YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_MOUNT_PATH, tmpPath); + conf.set(YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_HIERARCHY, + "/hadoop-yarn"); + File cpu = new File(new File(tmpPath, "cpuacct,cpu"), "/hadoop-yarn"); + + try { + Assert.assertTrue("temp dir should be created", cpu.mkdirs()); + + CGroupsHandlerImpl cGroupsHandler = new CGroupsHandlerImpl(conf, null); + cGroupsHandler.initializeCGroupController( + CGroupsHandler.CGroupController.CPU); + + Assert.assertEquals("CPU CGRoup path was not set", cpu.getAbsolutePath(), + new File(cGroupsHandler.getPathForCGroup( + CGroupsHandler.CGroupController.CPU, "")).getAbsolutePath()); + + } finally { + FileUtils.deleteQuietly(cpu); + } + } } \ No newline at end of file Index: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestCgroupsLCEResourcesHandler.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestCgroupsLCEResourcesHandler.java (date 1501180845000) +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestCgroupsLCEResourcesHandler.java (date 1501266963000) @@ -41,6 +41,8 @@ import java.util.Set; import java.util.concurrent.CountDownLatch; +import static org.mockito.Mockito.when; + @Deprecated public class TestCgroupsLCEResourcesHandler { private static File cgroupDir = null; @@ -388,4 +390,33 @@ FileUtils.deleteQuietly(memory); } } + + @Test + public void testManualCgroupSetting() throws IOException { + CgroupsLCEResourcesHandler handler = new CgroupsLCEResourcesHandler(); + YarnConfiguration conf = new YarnConfiguration(); + conf.set(YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_MOUNT_PATH, + cgroupDir.getAbsolutePath()); + handler.setConf(conf); + File cpu = new File(new File(cgroupDir, "cpuacct,cpu"), "/hadoop-yarn"); + + try { + Assert.assertTrue("temp dir should be created", cpu.mkdirs()); + + final int numProcessors = 4; + ResourceCalculatorPlugin plugin = + Mockito.mock(ResourceCalculatorPlugin.class); + Mockito.doReturn(numProcessors).when(plugin).getNumProcessors(); + Mockito.doReturn(numProcessors).when(plugin).getNumCores(); + when(plugin.getNumProcessors()).thenReturn(8); + handler.init(null, plugin); + + Assert.assertEquals("CPU CGRoup path was not set", cpu.getParent(), + handler.getControllerPaths().get("cpu")); + + } finally { + FileUtils.deleteQuietly(cpu); + } + } + } Index: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeManagerCgroups.md IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeManagerCgroups.md (date 1501180845000) +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeManagerCgroups.md (date 1501266963000) @@ -17,7 +17,7 @@ -CGroups is a mechanism for aggregating/partitioning sets of tasks, and all their future children, into hierarchical groups with specialized behaviour. CGroups is a Linux kernel feature and was merged into kernel version 2.6.24. From a YARN perspective, this allows containers to be limited in their resource usage. A good example of this is CPU usage. Without CGroups, it becomes hard to limit container CPU usage. Currently, CGroups is only used for limiting CPU usage. +CGroups is a mechanism for aggregating/partitioning sets of tasks, and all their future children, into hierarchical groups with specialized behaviour. CGroups is a Linux kernel feature and was merged into kernel version 2.6.24. From a YARN perspective, this allows containers to be limited in their resource usage. A good example of this is CPU usage. Without CGroups, it becomes hard to limit container CPU usage. CGroups Configuration --------------------- @@ -32,7 +32,7 @@ | `yarn.nodemanager.linux-container-executor.resources-handler.class` | This should be set to "org.apache.hadoop.yarn.server.nodemanager.util.CgroupsLCEResourcesHandler". Using the LinuxContainerExecutor doesn't force you to use CGroups. If you wish to use CGroups, the resource-handler-class must be set to CGroupsLCEResourceHandler. | | `yarn.nodemanager.linux-container-executor.cgroups.hierarchy` | The cgroups hierarchy under which to place YARN proccesses(cannot contain commas). If yarn.nodemanager.linux-container-executor.cgroups.mount is false (that is, if cgroups have been pre-configured) and the Yarn user has write access to the parent directory, then the directory will be created. If the directory already exists, the administrator has to give Yarn write permissions to it recursively. | | `yarn.nodemanager.linux-container-executor.cgroups.mount` | Whether the LCE should attempt to mount cgroups if not found - can be true or false. | -| `yarn.nodemanager.linux-container-executor.cgroups.mount-path` | Where the LCE should attempt to mount cgroups if not found. Common locations include /sys/fs/cgroup and /cgroup; the default location can vary depending on the Linux distribution in use. This path must exist before the NodeManager is launched. Only used when the LCE resources handler is set to the CgroupsLCEResourcesHandler, and yarn.nodemanager.linux-container-executor.cgroups.mount is true. A point to note here is that the container-executor binary will try to mount the path specified + "/" + the subsystem. In our case, since we are trying to limit CPU the binary tries to mount the path specified + "/cpu" and that's the path it expects to exist. | +| `yarn.nodemanager.linux-container-executor.cgroups.mount-path` | Optional. Where CGroups are located. LCE will try to mount them here, if `yarn.nodemanager.linux-container-executor.cgroups.mount` is true. LCE will try to use CGroups from this location, if `yarn.nodemanager.linux-container-executor.cgroups.mount` is false. If specified, this path and its subdirectories (CGroup hierarchies) must exist and they should be readable and writable by Yarn before the NodeManager is launched. See more details below. | | `yarn.nodemanager.linux-container-executor.group` | The Unix group of the NodeManager. It should match the setting in "container-executor.cfg". This configuration is required for validating the secure access of the container-executor binary. | The following settings are related to limiting resource usage of YARN containers: @@ -42,6 +42,17 @@ | `yarn.nodemanager.resource.percentage-physical-cpu-limit` | This setting lets you limit the cpu usage of all YARN containers. It sets a hard upper limit on the cumulative CPU usage of the containers. For example, if set to 60, the combined CPU usage of all YARN containers will not exceed 60%. | | `yarn.nodemanager.linux-container-executor.cgroups.strict-resource-usage` | CGroups allows cpu usage limits to be hard or soft. When this setting is true, containers cannot use more CPU usage than allocated even if spare CPU is available. This ensures that containers can only use CPU that they were allocated. When set to false, containers can use spare CPU if available. It should be noted that irrespective of whether set to true or false, at no time can the combined CPU usage of all containers exceed the value specified in "yarn.nodemanager.resource.percentage-physical-cpu-limit". | +CGroups mount options +--------------------- + +Yarn uses CGroups through a directory structure mounted into the file system by the kernel. There are three options to attach to CGroups. + +| Option | Description | +|:---- |:---- | +| Discover CGroups mounted already | This should be used on newer systems like RHEL7 or Ubuntu16 or if the administrator mounts CGroups before Yarn starts. Set `yarn.nodemanager.linux-container-executor.cgroups.mount` to false and leave other settings default. Yarn will locate the mount points in `/proc/mounts`. Common locations include `/sys/fs/cgroup` and `/cgroup`. The default location can vary depending on the Linux distribution in use.| +| Mount CGroups by Yarn | If the system does not have CGroups mounted or it is mounted to an inaccessible location then point `yarn.nodemanager.linux-container-executor.cgroups.mount-path` to an empty directory. Set `yarn.nodemanager.linux-container-executor.cgroups.mount` to true. A point to note here is that the container-executor binary will try to create and mount each subsystem as a subdirectory under this path. If `cpu` is already mounted somewhere with `cpuacct`, then the directory `cpu,cpuacct` will be created for the hierarchy.| +| CGroups mounted or linked already but not in `/proc/mounts` | If cgroups is accessible through lxcfs or simulated by another filesystem, then point `yarn.nodemanager.linux-container-executor.cgroups.mount-path` to your CGroups root directory. Set `yarn.nodemanager.linux-container-executor.cgroups.mount` to false. Yarn tries to use this path first, before any CGroup mount point discovery. The path should have a subdirectory for each CGroup hierarchy named by the comma separated CGroup subsystems supported like `/cpu,cpuacct`. Valid subsystem names are `cpu, cpuacct, cpuset, memory, net_cls, blkio, freezer, devices`.| + CGroups and security --------------------