diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 81c9cb2..96e9c19 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -1517,11 +1517,21 @@
- Where the LCE should attempt to mount cgroups if not found. Common locations
- include /sys/fs/cgroup and /cgroup; the default location can vary depending on the Linux
- distribution in use. This path must exist before the NodeManager is launched.
- Only used when the LCE resources handler is set to the CgroupsLCEResourcesHandler, and
- yarn.nodemanager.linux-container-executor.cgroups.mount is true.
+ Requested cgroup mount path. Yarn has built in functionality to discover
+ the system cgroup mount paths, so use this setting only, if the discovery does not work.
+
+ This path must exist before the NodeManager is launched.
+ The location can vary depending on the Linux distribution in use.
+ Common locations include /sys/fs/cgroup and /cgroup.
+
+ If cgroups are not mounted, set yarn.nodemanager.linux-container-executor.cgroups.mount
+ to true. In this case it specifies, where the LCE should attempt to mount cgroups if not found.
+
+ If cgroups is accessible through lxcfs or some other file systems,
+ then set this path and yarn.nodemanager.linux-container-executor.cgroups.mount to false.
+ Yarn tries to use this path first, before any cgroup mount point discovery.
+ Only used when the LCE resources handler is set to the CgroupsLCEResourcesHandler
+
yarn.nodemanager.linux-container-executor.cgroups.mount-path
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandler.java
index 8fc35a8..82bd366 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandler.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandler.java
@@ -23,6 +23,9 @@
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import java.util.HashSet;
+import java.util.Set;
+
/**
* Provides CGroups functionality. Implementations are expected to be
* thread-safe
@@ -54,6 +57,18 @@
String getName() {
return name;
}
+
+ /**
+ * Get the list of valid cgroup names.
+ * @return The set of cgroup name strings
+ */
+ public static Set getValidCGroups() {
+ HashSet validCgroups = new HashSet<>();
+ for (CGroupController controller : CGroupController.values()) {
+ validCgroups.add(controller.getName());
+ }
+ return validCgroups;
+ }
}
String CGROUP_FILE_TASKS = "tasks";
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandlerImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandlerImpl.java
index 85b01cd..77bb071 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandlerImpl.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandlerImpl.java
@@ -142,11 +142,18 @@ private void initializeControllerPaths() throws ResourceHandlerException {
// the same hierarchy will be mounted at each mount point with the same
// subsystem set.
- Map> newMtab;
+ Map> newMtab = null;
Map cPaths;
try {
- // parse mtab
- newMtab = parseMtab(mtabFile);
+ if (this.cGroupMountPath != null && !this.enableCGroupMount) {
+ newMtab = ResourceHandlerModule.
+ parseConfiguredCGroupPath(this.cGroupMountPath);
+ }
+
+ if (newMtab == null) {
+ // parse mtab
+ newMtab = parseMtab(mtabFile);
+ }
// find cgroup controller paths
cPaths = initializeControllerPathsFromMtab(newMtab);
@@ -203,10 +210,8 @@ private void initializeControllerPaths() throws ResourceHandlerException {
throws IOException {
Map> ret = new HashMap<>();
BufferedReader in = null;
- HashSet validCgroups = new HashSet<>();
- for (CGroupController controller : CGroupController.values()) {
- validCgroups.add(controller.getName());
- }
+ Set validCgroups =
+ CGroupsHandler.CGroupController.getValidCGroups();
try {
FileInputStream fis = new FileInputStream(new File(mtab));
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/ResourceHandlerModule.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/ResourceHandlerModule.java
index 7fc04bd..9e83c27 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/ResourceHandlerModule.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/ResourceHandlerModule.java
@@ -31,6 +31,13 @@
import org.apache.hadoop.yarn.server.nodemanager.util.CgroupsLCEResourcesHandler;
import org.apache.hadoop.yarn.server.nodemanager.util.DefaultLCEResourcesHandler;
+import java.io.File;
+import java.io.IOException;
+import java.util.Set;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.HashMap;
+import java.util.Arrays;
import java.util.ArrayList;
import java.util.List;
@@ -229,4 +236,45 @@ public static ResourceHandlerChain getConfiguredResourceHandlerChain(
static void nullifyResourceHandlerChain() throws ResourceHandlerException {
resourceHandlerChain = null;
}
+
+ /**
+ * If a cgroup mount directory is specified, it returns cgroup directories
+ * with valid names.
+ * The requirement is that each hierarchy has to be named with the comma
+ * separated names of subsystems supported.
+ * For example: /sys/fs/cgroup/cpu,cpuacct
+ * @param cgroupMountPath Root cgroup mount path (/sys/fs/cgroup in the
+ * example above)
+ * @return A path to cgroup subsystem set mapping in the same format as
+ * {@link CGroupsHandlerImpl#parseMtab(String)}
+ * @throws IOException if the specified directory cannot be listed
+ */
+ public static Map> parseConfiguredCGroupPath(
+ String cgroupMountPath) throws IOException {
+ File cgroupDir = new File(cgroupMountPath);
+ File[] list = cgroupDir.listFiles();
+ if (list == null) {
+ throw new IOException("Empty cgroup mount directory specified: " +
+ cgroupMountPath);
+ }
+
+ Map> pathSubsystemMappings = new HashMap<>();
+ Set validCGroups =
+ CGroupsHandler.CGroupController.getValidCGroups();
+ for (File candidate: list) {
+ Set cgroupList =
+ new HashSet<>(Arrays.asList(candidate.getName().split(",")));
+ // Collect the valid subsystem names
+ cgroupList.retainAll(validCGroups);
+ if (!cgroupList.isEmpty()) {
+ if (candidate.isDirectory() && candidate.canWrite()) {
+ pathSubsystemMappings.put(candidate.getAbsolutePath(), cgroupList);
+ } else {
+ LOG.warn("The following cgroup is not a directory or it is not"
+ + " writable" + candidate.getAbsolutePath());
+ }
+ }
+ }
+ return pathSubsystemMappings;
+ }
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/CgroupsLCEResourcesHandler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/CgroupsLCEResourcesHandler.java
index bca4fdc..48de453 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/CgroupsLCEResourcesHandler.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/CgroupsLCEResourcesHandler.java
@@ -27,6 +27,7 @@
import java.io.OutputStreamWriter;
import java.io.PrintWriter;
import java.io.Writer;
+import java.util.Arrays;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
@@ -39,7 +40,6 @@
import com.google.common.annotations.VisibleForTesting;
-import com.google.common.collect.Sets;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -51,6 +51,8 @@
import org.apache.hadoop.yarn.server.nodemanager.LinuxContainerExecutor;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperation;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.CGroupsCpuResourceHandlerImpl;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.CGroupsHandler;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerModule;
import org.apache.hadoop.yarn.util.Clock;
import org.apache.hadoop.yarn.util.ResourceCalculatorPlugin;
import org.apache.hadoop.yarn.util.SystemClock;
@@ -400,6 +402,8 @@ public String getResourcesOption(ContainerId containerId) {
private Map> parseMtab() throws IOException {
Map> ret = new HashMap>();
BufferedReader in = null;
+ Set validCgroups =
+ CGroupsHandler.CGroupController.getValidCGroups();
try {
FileInputStream fis = new FileInputStream(new File(getMtabFileName()));
@@ -415,8 +419,11 @@ public String getResourcesOption(ContainerId containerId) {
String options = m.group(3);
if (type.equals(CGROUPS_FSTYPE)) {
- HashSet value = Sets.newHashSet(options.split(","));
- ret.put(path, value);
+ Set cgroupList =
+ new HashSet<>(Arrays.asList(options.split(",")));
+ // Collect the valid subsystem names
+ cgroupList.retainAll(validCgroups);
+ ret.put(path, cgroupList);
}
}
}
@@ -448,7 +455,16 @@ String findControllerInMtab(String controller,
private void initializeControllerPaths() throws IOException {
String controllerPath;
- Map> parsedMtab = parseMtab();
+ Map> parsedMtab = null;
+
+ if (this.cgroupMountPath != null && !this.cgroupMount) {
+ parsedMtab = ResourceHandlerModule.
+ parseConfiguredCGroupPath(this.cgroupMountPath);
+ }
+
+ if (parsedMtab == null) {
+ parsedMtab = parseMtab();
+ }
// CPU
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsHandlerImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsHandlerImpl.java
index dd8e338..e678db0 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsHandlerImpl.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsHandlerImpl.java
@@ -573,4 +573,29 @@ public void testRemount()
new File(new File(newMountPoint, "cpu"), this.hierarchy);
assertTrue("Yarn cgroup should exist", hierarchyFile.exists());
}
+
+
+ @Test
+ public void testManualCgroupSetting() throws ResourceHandlerException {
+ YarnConfiguration conf = new YarnConfiguration();
+ conf.set(YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_MOUNT_PATH, tmpPath);
+ conf.set(YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_HIERARCHY,
+ "/hadoop-yarn");
+ File cpu = new File(new File(tmpPath, "cpuacct,cpu"), "/hadoop-yarn");
+
+ try {
+ Assert.assertTrue("temp dir should be created", cpu.mkdirs());
+
+ CGroupsHandlerImpl cGroupsHandler = new CGroupsHandlerImpl(conf, null);
+ cGroupsHandler.initializeCGroupController(
+ CGroupsHandler.CGroupController.CPU);
+
+ Assert.assertEquals(cpu.getAbsolutePath(),
+ new File(cGroupsHandler.getPathForCGroup(
+ CGroupsHandler.CGroupController.CPU, "")).getAbsolutePath());
+
+ } finally {
+ FileUtils.deleteQuietly(cpu);
+ }
+ }
}
\ No newline at end of file
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestCgroupsLCEResourcesHandler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestCgroupsLCEResourcesHandler.java
index b562133..7aa061d 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestCgroupsLCEResourcesHandler.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestCgroupsLCEResourcesHandler.java
@@ -41,6 +41,8 @@
import java.util.Set;
import java.util.concurrent.CountDownLatch;
+import static org.mockito.Mockito.when;
+
@Deprecated
public class TestCgroupsLCEResourcesHandler {
private static File cgroupDir = null;
@@ -388,4 +390,33 @@ public void testSelectCgroup() {
FileUtils.deleteQuietly(memory);
}
}
+
+ @Test
+ public void testManualCgroupSetting() throws IOException {
+ CgroupsLCEResourcesHandler handler = new CgroupsLCEResourcesHandler();
+ YarnConfiguration conf = new YarnConfiguration();
+ conf.set(YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_MOUNT_PATH,
+ cgroupDir.getAbsolutePath());
+ handler.setConf(conf);
+ File cpu = new File(new File(cgroupDir, "cpuacct,cpu"), "/hadoop-yarn");
+
+ try {
+ Assert.assertTrue("temp dir should be created", cpu.mkdirs());
+
+ final int numProcessors = 4;
+ ResourceCalculatorPlugin plugin =
+ Mockito.mock(ResourceCalculatorPlugin.class);
+ Mockito.doReturn(numProcessors).when(plugin).getNumProcessors();
+ Mockito.doReturn(numProcessors).when(plugin).getNumCores();
+ when(plugin.getNumProcessors()).thenReturn(8);
+ handler.init(null, plugin);
+
+ Assert.assertEquals(cpu.getParent(),
+ handler.getControllerPaths().get("cpu"));
+
+ } finally {
+ FileUtils.deleteQuietly(cpu);
+ }
+ }
+
}