diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index e687eef370..407660f4e0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -1509,11 +1509,22 @@
- Where the LCE should attempt to mount cgroups if not found. Common locations
- include /sys/fs/cgroup and /cgroup; the default location can vary depending on the Linux
- distribution in use. This path must exist before the NodeManager is launched.
- Only used when the LCE resources handler is set to the CgroupsLCEResourcesHandler, and
- yarn.nodemanager.linux-container-executor.cgroups.mount is true.
+ Requested cgroup mount path. Yarn has built in functionality to discover
+ the system cgroup mount paths, so use this setting only, if the discovery does not work.
+
+ This path must exist before the NodeManager is launched.
+ The location can vary depending on the Linux distribution in use.
+ Common locations include /sys/fs/cgroup and /cgroup.
+
+ If cgroups are not mounted, set yarn.nodemanager.linux-container-executor.cgroups.mount
+ to true. In this case it specifies, where the LCE should attempt to mount cgroups if not found.
+
+ If cgroups is accessible through lxcfs or some other file system,
+ then set this path and yarn.nodemanager.linux-container-executor.cgroups.mount to false.
+ Yarn tries to use this path first, before any cgroup mount point discovery.
+ If it cannot find this directory, it falls back to searching for cgroup mount points in the system.
+ Only used when the LCE resources handler is set to the CgroupsLCEResourcesHandler
+
yarn.nodemanager.linux-container-executor.cgroups.mount-path
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandler.java
index 8fc35a8232..0691a6b951 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandler.java
@@ -23,6 +23,9 @@
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import java.util.HashSet;
+import java.util.Set;
+
/**
* Provides CGroups functionality. Implementations are expected to be
* thread-safe
@@ -54,6 +57,18 @@
String getName() {
return name;
}
+
+ /**
+ * Get the list of valid cgroup names
+ * @return The set of cgroup name strings
+ */
+ public static Set getValidCGroups() {
+ HashSet validCgroups = new HashSet<>();
+ for (CGroupController controller : CGroupController.values()) {
+ validCgroups.add(controller.getName());
+ }
+ return validCgroups;
+ }
}
String CGROUP_FILE_TASKS = "tasks";
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandlerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandlerImpl.java
index 85b01cd1e8..5f45fa6edd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandlerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandlerImpl.java
@@ -142,11 +142,17 @@ private void initializeControllerPaths() throws ResourceHandlerException {
// the same hierarchy will be mounted at each mount point with the same
// subsystem set.
- Map> newMtab;
+ Map> newMtab = null;
Map cPaths;
try {
- // parse mtab
- newMtab = parseMtab(mtabFile);
+ if (this.cGroupMountPath != null && !this.enableCGroupMount) {
+ newMtab = checkConfiguredCGroupPath(this.cGroupMountPath);
+ }
+
+ if (newMtab == null) {
+ // parse mtab
+ newMtab = parseMtab(mtabFile);
+ }
// find cgroup controller paths
cPaths = initializeControllerPathsFromMtab(newMtab);
@@ -183,6 +189,40 @@ private void initializeControllerPaths() throws ResourceHandlerException {
return ret;
}
+ /**
+ * If a cgroup mount directory is specified, it returns cgroup directories with valid names
+ * The requirement is that each hierarchy has to be named with the comma separated names of subsystems supported
+ * For example: /sys/fs/cgroup/cpu,cpuacct
+ * @param cGroupMountPathSpecified Root cgroup mount path (/sys/fs/cgroup in the example above)
+ * @return A path to cgroup subsystem set mapping in the same format as {@link #parseMtab(String)}
+ * @throws IOException if the specified directory cannot be listed
+ */
+ private static Map> checkConfiguredCGroupPath(String cGroupMountPathSpecified)
+ throws IOException {
+ Set validCgroups = CGroupsHandler.CGroupController.getValidCGroups();
+ Map> ret = new HashMap<>();
+ File cgroupDir = new File(cGroupMountPathSpecified);
+ File[] list = cgroupDir.listFiles();
+
+ if (list == null) {
+ throw new IOException("Empty cgroup mount directory specified: " + cGroupMountPathSpecified);
+ }
+ for (File candidate: list) {
+ Set cgroupList =
+ new HashSet<>(Arrays.asList(candidate.getName().split(",")));
+ // Collect the valid subsystem names
+ cgroupList.retainAll(validCgroups);
+ if (!cgroupList.isEmpty()) {
+ if (candidate.isDirectory() && candidate.canWrite()) {
+ ret.put(candidate.getAbsolutePath(), cgroupList);
+ } else {
+ LOG.warn("The following cgroup is not a directory or it is not writable" + candidate.getAbsolutePath());
+ }
+ }
+ }
+ return ret;
+ }
+
/* We are looking for entries of the form:
* none /cgroup/path/mem cgroup rw,memory 0 0
*
@@ -203,10 +243,7 @@ private void initializeControllerPaths() throws ResourceHandlerException {
throws IOException {
Map> ret = new HashMap<>();
BufferedReader in = null;
- HashSet validCgroups = new HashSet<>();
- for (CGroupController controller : CGroupController.values()) {
- validCgroups.add(controller.getName());
- }
+ Set validCgroups = CGroupsHandler.CGroupController.getValidCGroups();
try {
FileInputStream fis = new FileInputStream(new File(mtab));
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/CgroupsLCEResourcesHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/CgroupsLCEResourcesHandler.java
index bca4fdc8c9..8e4dce6942 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/CgroupsLCEResourcesHandler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/CgroupsLCEResourcesHandler.java
@@ -27,6 +27,7 @@
import java.io.OutputStreamWriter;
import java.io.PrintWriter;
import java.io.Writer;
+import java.util.Arrays;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
@@ -51,6 +52,7 @@
import org.apache.hadoop.yarn.server.nodemanager.LinuxContainerExecutor;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperation;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.CGroupsCpuResourceHandlerImpl;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.CGroupsHandler;
import org.apache.hadoop.yarn.util.Clock;
import org.apache.hadoop.yarn.util.ResourceCalculatorPlugin;
import org.apache.hadoop.yarn.util.SystemClock;
@@ -400,6 +402,7 @@ public String getResourcesOption(ContainerId containerId) {
private Map> parseMtab() throws IOException {
Map> ret = new HashMap>();
BufferedReader in = null;
+ Set validCgroups = CGroupsHandler.CGroupController.getValidCGroups();
try {
FileInputStream fis = new FileInputStream(new File(getMtabFileName()));
@@ -415,8 +418,11 @@ public String getResourcesOption(ContainerId containerId) {
String options = m.group(3);
if (type.equals(CGROUPS_FSTYPE)) {
- HashSet value = Sets.newHashSet(options.split(","));
- ret.put(path, value);
+ Set cgroupList =
+ new HashSet<>(Arrays.asList(options.split(",")));
+ // Collect the valid subsystem names
+ cgroupList.retainAll(validCgroups);
+ ret.put(path, cgroupList);
}
}
}
@@ -446,9 +452,51 @@ String findControllerInMtab(String controller,
return null;
}
+ /**
+ * If a cgroup mount directory is specified, it returns cgroup directories with valid names
+ * The requirement is that each hierarchy has to be named with the comma separated names of subsystems supported
+ * For example: /sys/fs/cgroup/cpu,cpuacct
+ * @param cGroupMountPathSpecified Root cgroup mount path (/sys/fs/cgroup in the example above)
+ * @return A path to cgroup subsystem set mapping in the same format as {@link #parseMtab()}
+ * @throws IOException if the specified directory cannot be listed
+ */
+ private static Map> checkConfiguredCGroupPath(String cGroupMountPathSpecified)
+ throws IOException {
+ Set validCgroups = CGroupsHandler.CGroupController.getValidCGroups();
+ Map> ret = new HashMap<>();
+ File cgroupDir = new File(cGroupMountPathSpecified);
+ File[] list = cgroupDir.listFiles();
+
+ if (list == null) {
+ throw new IOException("Empty cgroup mount directory specified: " + cGroupMountPathSpecified);
+ }
+ for (File candidate: list) {
+ Set cgroupList =
+ new HashSet<>(Arrays.asList(candidate.getName().split(",")));
+ // Collect the valid subsystem names
+ cgroupList.retainAll(validCgroups);
+ if (!cgroupList.isEmpty()) {
+ if (candidate.isDirectory() && candidate.canWrite()) {
+ ret.put(candidate.getAbsolutePath(), cgroupList);
+ } else {
+ LOG.warn("The following cgroup is not a directory or it is not writable" + candidate.getAbsolutePath());
+ }
+ }
+ }
+ return ret;
+ }
+
private void initializeControllerPaths() throws IOException {
String controllerPath;
- Map> parsedMtab = parseMtab();
+ Map> parsedMtab = null;
+
+ if (this.cgroupMountPath != null && !this.cgroupMount) {
+ parsedMtab = checkConfiguredCGroupPath(this.cgroupMountPath);
+ }
+
+ if (parsedMtab == null) {
+ parsedMtab = parseMtab();
+ }
// CPU
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsHandlerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsHandlerImpl.java
index dd8e33838b..c9d7b38406 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsHandlerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsHandlerImpl.java
@@ -573,4 +573,27 @@ public void testRemount()
new File(new File(newMountPoint, "cpu"), this.hierarchy);
assertTrue("Yarn cgroup should exist", hierarchyFile.exists());
}
+
+
+ @Test
+ public void testManualCgroupSetting() throws ResourceHandlerException {
+ YarnConfiguration conf = new YarnConfiguration();
+ conf.set(YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_MOUNT_PATH, tmpPath);
+ conf.set(YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_HIERARCHY, "/hadoop-yarn");
+ File cpu = new File(new File(tmpPath, "cpuacct,cpu"), "/hadoop-yarn");
+
+ try {
+ Assert.assertTrue("temp dir should be created", cpu.mkdirs());
+
+ CGroupsHandlerImpl cGroupsHandler = new CGroupsHandlerImpl(conf, null);
+ cGroupsHandler.initializeCGroupController(
+ CGroupsHandler.CGroupController.CPU);
+
+ Assert.assertEquals(cpu.getAbsolutePath(),
+ new File(cGroupsHandler.getPathForCGroup(CGroupsHandler.CGroupController.CPU, "")).getAbsolutePath());
+
+ } finally {
+ FileUtils.deleteQuietly(cpu);
+ }
+ }
}
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestCgroupsLCEResourcesHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestCgroupsLCEResourcesHandler.java
index b562133eed..186b814ac1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestCgroupsLCEResourcesHandler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestCgroupsLCEResourcesHandler.java
@@ -41,6 +41,8 @@
import java.util.Set;
import java.util.concurrent.CountDownLatch;
+import static org.mockito.Mockito.when;
+
@Deprecated
public class TestCgroupsLCEResourcesHandler {
private static File cgroupDir = null;
@@ -388,4 +390,31 @@ public void testSelectCgroup() {
FileUtils.deleteQuietly(memory);
}
}
+
+ @Test
+ public void testManualCgroupSetting() throws IOException {
+ CgroupsLCEResourcesHandler handler = new CgroupsLCEResourcesHandler();
+ YarnConfiguration conf = new YarnConfiguration();
+ conf.set(YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_MOUNT_PATH, cgroupDir.getAbsolutePath());
+ handler.setConf(conf);
+ File cpu = new File(new File(cgroupDir, "cpuacct,cpu"), "/hadoop-yarn");
+
+ try {
+ Assert.assertTrue("temp dir should be created", cpu.mkdirs());
+
+ final int numProcessors = 4;
+ ResourceCalculatorPlugin plugin =
+ Mockito.mock(ResourceCalculatorPlugin.class);
+ Mockito.doReturn(numProcessors).when(plugin).getNumProcessors();
+ Mockito.doReturn(numProcessors).when(plugin).getNumCores();
+ when(plugin.getNumProcessors()).thenReturn(8);
+ handler.init(null, plugin);
+
+ Assert.assertEquals(cpu.getParent(), handler.getControllerPaths().get("cpu"));
+
+ } finally {
+ FileUtils.deleteQuietly(cpu);
+ }
+ }
+
}