diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsControllerPaths.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsControllerPaths.java new file mode 100644 index 00000000000..11229556879 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsControllerPaths.java @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.Maps; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.CGroupsHandler.CGroupController; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.util.List; +import java.util.Map; + +/** + * Stores cgroups paths for all {@link CGroupController}. + */ +public class CGroupsControllerPaths { + private static final Logger LOG = + LoggerFactory.getLogger(CGroupsControllerPaths.class); + + private final Map controllerPaths = + Maps.newHashMap(); + + public void storePath(CGroupController controller, String path) { + controllerPaths.put(controller, path); + } + + public String getPath(CGroupController controller) { + return controllerPaths.get(controller); + } + + public boolean hasPath(CGroupController controller) { + return controllerPaths.containsKey(controller); + } + + public boolean isEmpty() { + return controllerPaths.isEmpty(); + } + + public int getNumberOfPaths() { + return controllerPaths.size(); + } + + @VisibleForTesting + static CGroupsControllerPaths initializeFromMountConfig( + CGroupsMountConfig mountConfig) { + CGroupsControllerPaths controllerPaths = new CGroupsControllerPaths(); + + for (CGroupController controller : + CGroupController.values()) { + String controllerName = controller.getName(); + String controllerPath = findControllerPathInMountConfig(controllerName, + mountConfig); + + if (controllerPath != null) { + controllerPaths.storePath(controller, controllerPath); + } + } + return controllerPaths; + } + + /** + * Find the hierarchy of the controller. + * The kernel ensures that a controller can only + * be part of a single hierarchy. + * The controller can be part of multiple mount points, if they belong to the + * same hierarchy. + * @param controller subsystem like cpu, cpuset, etc... + * @param mountConfig stores map of paths to controllers + * @return The first mount path that has the requested controller and + * is also readable. If no path found for a controller, + * the return value is null. + */ + @VisibleForTesting + public static String findControllerPathInMountConfig(String controller, + CGroupsMountConfig mountConfig) { + List paths = mountConfig.getPathsForController(controller); + if (paths != null) { + for (String path : paths) { + if (new File(path).canRead()) { + return path; + } else { + LOG.warn(String.format( + "Skipping inaccessible cgroup mount point %s", path)); + } + } + } + return null; + } + + @Override + public String toString() { + return "CGroupsControllerPaths{" + + "controllerPaths=" + controllerPaths + + '}'; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandlerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandlerImpl.java index fab14908332..48f39710608 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandlerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandlerImpl.java @@ -29,8 +29,6 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.io.IOUtils; -import org.apache.hadoop.util.Shell; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperation; import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationException; @@ -41,15 +39,8 @@ import java.io.*; import java.nio.file.Files; import java.nio.file.Paths; -import java.util.Arrays; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; -import java.util.regex.Matcher; -import java.util.regex.Pattern; /** * Support for interacting with various CGroup subsystems. Thread-safe. @@ -62,7 +53,7 @@ private static final Logger LOG = LoggerFactory.getLogger(CGroupsHandlerImpl.class); private static final String MTAB_FILE = "/proc/mounts"; - private static final String CGROUPS_FSTYPE = "cgroup"; + private final String mtabFile; private final String cGroupPrefix; @@ -70,8 +61,8 @@ private final String cGroupMountPath; private final long deleteCGroupTimeout; private final long deleteCGroupDelay; - private Map controllerPaths; - private Map> parsedMtab; + private final CGroupsControllerPaths controllerPaths; + private final CGroupsMountConfig cGroupsMountConfig; private final ReadWriteLock rwLock; private final PrivilegedOperationExecutor privilegedOperationExecutor; private final Clock clock; @@ -84,6 +75,7 @@ * @param mtab mount file location * @throws ResourceHandlerException if initialization failed */ + @VisibleForTesting CGroupsHandlerImpl(Configuration conf, PrivilegedOperationExecutor privilegedOperationExecutor, String mtab) throws ResourceHandlerException { @@ -103,13 +95,22 @@ this.deleteCGroupDelay = conf.getLong(YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_DELETE_DELAY, YarnConfiguration.DEFAULT_NM_LINUX_CONTAINER_CGROUPS_DELETE_DELAY); - this.controllerPaths = new HashMap<>(); - this.parsedMtab = new HashMap<>(); this.rwLock = new ReentrantReadWriteLock(); this.privilegedOperationExecutor = privilegedOperationExecutor; this.clock = SystemClock.getInstance(); - mtabFile = mtab; - init(); + this.mtabFile = mtab; + CGroupsMountConfig mountConfig = initializeMountConfig(); + CGroupsControllerPaths ctrlPaths = + CGroupsControllerPaths.initializeFromMountConfig(mountConfig); + + // we want to do a bulk update without the paths changing concurrently + rwLock.writeLock().lock(); + try { + this.controllerPaths = ctrlPaths; + this.cGroupsMountConfig = mountConfig; + } finally { + rwLock.writeLock().unlock(); + } } /** @@ -124,160 +125,41 @@ this(conf, privilegedOperationExecutor, MTAB_FILE); } - private void init() throws ResourceHandlerException { - initializeControllerPaths(); - } - @Override public String getControllerPath(CGroupController controller) { rwLock.readLock().lock(); try { - return controllerPaths.get(controller); + return controllerPaths.getPath(controller); } finally { rwLock.readLock().unlock(); } } - private void initializeControllerPaths() throws ResourceHandlerException { - // Cluster admins may have some subsystems mounted in specific locations - // We'll attempt to figure out mount points. We do this even if we plan - // to mount cgroups into our own tree to control the path permissions or - // to mount subsystems that are not mounted previously. - // The subsystems for new and existing mount points have to match, and - // the same hierarchy will be mounted at each mount point with the same - // subsystem set. - - Map> newMtab = null; - Map cPaths; + /** + * Cluster admins may have some subsystems mounted in specific locations + * We'll attempt to figure out mount points. We do this even if we plan + * to mount cgroups into our own tree to control the path permissions or + * to mount subsystems that are not mounted previously. + * The subsystems for new and existing mount points have to match, and + * the same hierarchy will be mounted at each mount point with the same + * subsystem set. + * @throws ResourceHandlerException + */ + private CGroupsMountConfig initializeMountConfig() + throws ResourceHandlerException { try { if (this.cGroupMountPath != null && !this.enableCGroupMount) { - newMtab = ResourceHandlerModule. + return CgroupsFileSystemMountParser. parseConfiguredCGroupPath(this.cGroupMountPath); + } else { + return MtabFileParser.parse(mtabFile); } - - if (newMtab == null) { - // parse mtab - newMtab = parseMtab(mtabFile); - } - - // find cgroup controller paths - cPaths = initializeControllerPathsFromMtab(newMtab); } catch (IOException e) { - LOG.warn("Failed to initialize controller paths! Exception: " + e); + LOG.warn("Failed to initialize mount config for controller paths! " + + "Exception: " + e); throw new ResourceHandlerException( - "Failed to initialize controller paths!"); - } - - // we want to do a bulk update without the paths changing concurrently - rwLock.writeLock().lock(); - try { - controllerPaths = cPaths; - parsedMtab = newMtab; - } finally { - rwLock.writeLock().unlock(); - } - } - - @VisibleForTesting - static Map initializeControllerPathsFromMtab( - Map> parsedMtab) - throws ResourceHandlerException { - Map ret = new HashMap<>(); - - for (CGroupController controller : CGroupController.values()) { - String subsystemName = controller.getName(); - String controllerPath = findControllerInMtab(subsystemName, parsedMtab); - - if (controllerPath != null) { - ret.put(controller, controllerPath); - } + "Failed to initialize controller paths!", e); } - return ret; - } - - /* We are looking for entries of the form: - * none /cgroup/path/mem cgroup rw,memory 0 0 - * - * Use a simple pattern that splits on the five spaces, and - * grabs the 2, 3, and 4th fields. - */ - - private static final Pattern MTAB_FILE_FORMAT = Pattern.compile( - "^[^\\s]+\\s([^\\s]+)\\s([^\\s]+)\\s([^\\s]+)\\s[^\\s]+\\s[^\\s]+$"); - - /* - * Returns a map: path -> mount options - * for mounts with type "cgroup". Cgroup controllers will - * appear in the list of options for a path. - */ - @VisibleForTesting - static Map> parseMtab(String mtab) - throws IOException { - Map> ret = new HashMap<>(); - BufferedReader in = null; - Set validCgroups = - CGroupsHandler.CGroupController.getValidCGroups(); - - try { - FileInputStream fis = new FileInputStream(new File(mtab)); - in = new BufferedReader(new InputStreamReader(fis, "UTF-8")); - - for (String str = in.readLine(); str != null; - str = in.readLine()) { - Matcher m = MTAB_FILE_FORMAT.matcher(str); - boolean mat = m.find(); - if (mat) { - String path = m.group(1); - String type = m.group(2); - String options = m.group(3); - - if (type.equals(CGROUPS_FSTYPE)) { - Set cgroupList = - new HashSet<>(Arrays.asList(options.split(","))); - // Collect the valid subsystem names - cgroupList.retainAll(validCgroups); - ret.put(path, cgroupList); - } - } - } - } catch (IOException e) { - if (Shell.LINUX) { - throw new IOException("Error while reading " + mtab, e); - } else { - // Ignore the error, if we are running on an os other than Linux - LOG.warn("Error while reading " + mtab, e); - } - } finally { - IOUtils.cleanupWithLogger(LOG, in); - } - - return ret; - } - - /** - * Find the hierarchy of the subsystem. - * The kernel ensures that a subsystem can only be part of a single hierarchy. - * The subsystem can be part of multiple mount points, if they belong to the - * same hierarchy. - * @param controller subsystem like cpu, cpuset, etc... - * @param entries map of paths to mount options - * @return the first mount path that has the requested subsystem - */ - @VisibleForTesting - static String findControllerInMtab(String controller, - Map> entries) { - for (Map.Entry> e : entries.entrySet()) { - if (e.getValue().contains(controller)) { - if (new File(e.getKey()).canRead()) { - return e.getKey(); - } else { - LOG.warn(String.format( - "Skipping inaccessible cgroup mount point %s", e.getKey())); - } - } - } - - return null; } private void mountCGroupController(CGroupController controller) @@ -301,8 +183,8 @@ private void mountCGroupController(CGroupController controller) // the operation will fail String mountOptions; if (existingMountPath != null) { - mountOptions = Joiner.on(',') - .join(parsedMtab.get(existingMountPath)); + mountOptions = Joiner.on(',').join( + cGroupsMountConfig.getControllersForPath(existingMountPath)); } else { mountOptions = controller.getName(); } @@ -319,7 +201,7 @@ private void mountCGroupController(CGroupController controller) privilegedOperationExecutor.executePrivilegedOperation(op, false); //if privileged operation succeeds, update controller paths - controllerPaths.put(controller, requestedMountPath); + controllerPaths.storePath(controller, requestedMountPath); } catch (PrivilegedOperationException e) { LOG.error("Failed to mount controller: " + controller.getName()); throw new ResourceHandlerException("Failed to mount controller: " @@ -372,7 +254,7 @@ public void initializeCGroupController(CGroupController controller) throws mountCGroupController(controller); } - // We are working with a pre-mounted contoller + // We are working with a pre-mounted controller // Make sure that YARN cgroup hierarchy path exists initializePreMountedCGroupController(controller); } @@ -614,6 +496,11 @@ public String getCGroupMountPath() { return cGroupMountPath; } + @VisibleForTesting + public CGroupsControllerPaths getControllerPaths() { + return controllerPaths; + } + @Override public String toString() { return CGroupsHandlerImpl.class.getName() + "{" + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsMountConfig.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsMountConfig.java new file mode 100644 index 00000000000..453e72d9cbc --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsMountConfig.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * Stores a mapping between cgroup paths and controller names. + * Keys will be paths and values will be controller (subsystem) names. + * In practice, most of the controllers have one path, example: + * devices -> /sys/fs/cgroup/devices + * But on some systems, cpu and cpuacct is mounted to one path, so both + * cpu and cpuacct controllers will have a mapping to the same path. Example: + * cpu -> /sys/fs/cgroup/cpu,cpuacct + * cpuacct -> /sys/fs/cgroup/cpu,cpuacct + */ +public class CGroupsMountConfig { + private Map> mappings = new HashMap<>(); + + public void mapPathToControllers(String path, Set controllers) { + mappings.put(path, controllers); + } + + public Set getControllersForPath(String path) { + return mappings.get(path); + } + + public List getPathsForController(String controller) { + return mappings.entrySet().stream() + .filter(e -> e.getValue().contains(controller)) + .map(Map.Entry::getKey) + .collect(Collectors.toList()); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CgroupsFileSystemMountParser.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CgroupsFileSystemMountParser.java new file mode 100644 index 00000000000..bab9cf90210 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CgroupsFileSystemMountParser.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; + +/** + * Responsible to parse cgroups controller info from a given path and + * to procude a {@link CGroupsMountConfig} as a result. + */ +public final class CgroupsFileSystemMountParser { + private static final Logger LOG = + LoggerFactory.getLogger(CgroupsFileSystemMountParser.class); + + private static final Set VALID_CGROUPS = + CGroupsHandler.CGroupController.getValidCGroups(); + + private CgroupsFileSystemMountParser() {} + + /** + * If a cgroup mount directory is specified, it returns cgroup directories + * with valid names. + * The requirement is that each hierarchy has to be named with the comma + * separated names of subsystems supported. + * For example: /sys/fs/cgroup/cpu,cpuacct + * @param cgroupMountPath Root cgroup mount path (/sys/fs/cgroup in the + * example above) + * @return A path to cgroup subsystem set mapping as an object of type + * {@link CGroupsMountConfig} + * @throws IOException if the specified directory cannot be listed + */ + public static CGroupsMountConfig parseConfiguredCGroupPath( + String cgroupMountPath) throws IOException { + File cgroupDir = new File(cgroupMountPath); + File[] cgroupFiles = cgroupDir.listFiles(); + if (cgroupFiles == null) { + throw new IOException("Empty cgroup mount directory specified: " + + cgroupMountPath); + } + + final CGroupsMountConfig mountConfig = new CGroupsMountConfig(); + for (File candidate: cgroupFiles) { + Set cgroupList = + new HashSet<>(Arrays.asList(candidate.getName().split(","))); + // Collect the valid subsystem names + cgroupList.retainAll(VALID_CGROUPS); + if (!cgroupList.isEmpty()) { + if (candidate.isDirectory()) { + mountConfig.mapPathToControllers(candidate.getAbsolutePath(), + cgroupList); + } else { + LOG.warn("The following cgroup is not a directory " + + candidate.getAbsolutePath()); + } + } + } + return mountConfig; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/MtabFileParser.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/MtabFileParser.java new file mode 100644 index 00000000000..b5eb7431a80 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/MtabFileParser.java @@ -0,0 +1,124 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources; + +import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.io.IOUtils; +import org.apache.hadoop.util.Shell; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.CGroupsHandler.CGroupController; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Responsible to parse cgroups controller info from a given mtab file + * (usually stores under /etc/mtab) and to procude a + * {@link CGroupsMountConfig} as a result. + */ +public final class MtabFileParser { + private static final Logger LOG = + LoggerFactory.getLogger(MtabFileParser.class); + + private static final String CGROUPS_FSTYPE = "cgroup"; + public static final String FIELD = "[^\\s]+"; + public static final String CAPTURED_FIELD = "([^\\s]+)"; + public static final String WHITESPACE = "\\s"; + + /** We are looking for entries of the form: + * none /cgroup/path/mem cgroup rw,memory 0 0 + * + * Use a simple pattern that splits on the five spaces, and + * grabs the 2, 3, and 4th fields. + */ + private static final String MTAB_LINE_PATTERN = new StringBuilder("^") + .append(FIELD).append(WHITESPACE) + .append(CAPTURED_FIELD).append(WHITESPACE) + .append(CAPTURED_FIELD).append(WHITESPACE) + .append(CAPTURED_FIELD).append(WHITESPACE) + .append(FIELD).append(WHITESPACE) + .append(FIELD).append("$").toString(); + private static final Pattern MTAB_FILE_FORMAT = + Pattern.compile(MTAB_LINE_PATTERN); + + private static final Set VALID_CGROUPS = + CGroupController.getValidCGroups(); + + private MtabFileParser() {} + + /** + * Parses an mtab file, usually stored under /etc/mtab + * Cgroup controllers will appear in the list of options for a path. + * + * Example lines of valid cgroup configs coming from an mtab file:
+ * cgroup /sys/fs/cgroup/memory cgroup rw,relatime,memory 0 0 + * cgroup /sys/fs/cgroup/cpuset cgroup rw,relatime,cpuset 0 0 + * cgroup /sys/fs/cgroup/devices cgroup rw,relatime,devices 0 0 + */ + @VisibleForTesting + public static CGroupsMountConfig parse(String mtabFile) + throws IOException { + final CGroupsMountConfig mountConfig = new CGroupsMountConfig(); + + BufferedReader in = null; + try { + FileInputStream fis = new FileInputStream(new File(mtabFile)); + in = new BufferedReader(new InputStreamReader(fis, + StandardCharsets.UTF_8)); + + for (String str = in.readLine(); str != null; str = in.readLine()) { + Matcher m = MTAB_FILE_FORMAT.matcher(str); + boolean matched = m.find(); + if (matched) { + String path = m.group(1); + String type = m.group(2); + String optionsStr = m.group(3); + + if (type.equals(CGROUPS_FSTYPE)) { + Set options = + new HashSet<>(Arrays.asList(optionsStr.split(","))); + // Collect the valid controller names + options.retainAll(VALID_CGROUPS); + mountConfig.mapPathToControllers(path, options); + } + } + } + } catch (IOException e) { + if (Shell.LINUX) { + throw new IOException("Error while reading " + mtabFile, e); + } else { + // Ignore the error, if we are running on an OS other than Linux + LOG.warn("Error while reading " + mtabFile, e); + } + } finally { + IOUtils.cleanupWithLogger(LOG, in); + } + + return mountConfig; + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/ResourceHandlerModule.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/ResourceHandlerModule.java index 2019417dd88..f752c527f72 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/ResourceHandlerModule.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/ResourceHandlerModule.java @@ -35,15 +35,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.File; -import java.io.IOException; import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Set; /** * Provides mechanisms to get various resource handlers - cpu, memory, network, @@ -353,45 +347,4 @@ public static ResourceHandlerChain getConfiguredResourceHandlerChain( static void nullifyResourceHandlerChain() throws ResourceHandlerException { resourceHandlerChain = null; } - - /** - * If a cgroup mount directory is specified, it returns cgroup directories - * with valid names. - * The requirement is that each hierarchy has to be named with the comma - * separated names of subsystems supported. - * For example: /sys/fs/cgroup/cpu,cpuacct - * @param cgroupMountPath Root cgroup mount path (/sys/fs/cgroup in the - * example above) - * @return A path to cgroup subsystem set mapping in the same format as - * {@link CGroupsHandlerImpl#parseMtab(String)} - * @throws IOException if the specified directory cannot be listed - */ - public static Map> parseConfiguredCGroupPath( - String cgroupMountPath) throws IOException { - File cgroupDir = new File(cgroupMountPath); - File[] list = cgroupDir.listFiles(); - if (list == null) { - throw new IOException("Empty cgroup mount directory specified: " + - cgroupMountPath); - } - - Map> pathSubsystemMappings = new HashMap<>(); - Set validCGroups = - CGroupsHandler.CGroupController.getValidCGroups(); - for (File candidate: list) { - Set cgroupList = - new HashSet<>(Arrays.asList(candidate.getName().split(","))); - // Collect the valid subsystem names - cgroupList.retainAll(validCGroups); - if (!cgroupList.isEmpty()) { - if (candidate.isDirectory()) { - pathSubsystemMappings.put(candidate.getAbsolutePath(), cgroupList); - } else { - LOG.warn("The following cgroup is not a directory " + - candidate.getAbsolutePath()); - } - } - } - return pathSubsystemMappings; - } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/CgroupsLCEResourcesHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/CgroupsLCEResourcesHandler.java index 6025260c4bd..477e21f96e3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/CgroupsLCEResourcesHandler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/CgroupsLCEResourcesHandler.java @@ -27,16 +27,13 @@ import java.io.OutputStreamWriter; import java.io.PrintWriter; import java.io.Writer; -import java.util.Arrays; import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Set; -import java.util.regex.Matcher; -import java.util.regex.Pattern; + +import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.CGroupsControllerPaths; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.CGroupsHandler.CGroupController; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.CGroupsMountConfig; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.CgroupsFileSystemMountParser; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.MtabFileParser; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -44,19 +41,17 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileUtil; -import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.nodemanager.LinuxContainerExecutor; import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperation; import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.CGroupsCpuResourceHandlerImpl; -import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.CGroupsHandler; -import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerModule; import org.apache.hadoop.yarn.util.Clock; import org.apache.hadoop.yarn.util.ResourceCalculatorPlugin; import org.apache.hadoop.yarn.util.SystemClock; + /** * Resource handler that lets you setup cgroups * to to handle cpu isolation. Please look at the ResourceHandlerModule @@ -85,7 +80,7 @@ private final String CPU_PERIOD_US = "cfs_period_us"; private final String CPU_QUOTA_US = "cfs_quota_us"; private final int CPU_DEFAULT_WEIGHT = 1024; // set by kernel - private final Map controllerPaths; // Controller -> path + private final CGroupsControllerPaths controllerPaths; private long deleteCgroupTimeout; private long deleteCgroupDelay; @@ -96,7 +91,7 @@ private int nodeVCores; public CgroupsLCEResourcesHandler() { - this.controllerPaths = new HashMap(); + this.controllerPaths = new CGroupsControllerPaths(); clock = SystemClock.getInstance(); } @@ -198,7 +193,8 @@ boolean isCpuWeightEnabled() { */ private String pathForCgroup(String controller, String groupName) { - String controllerPath = controllerPaths.get(controller); + String controllerPath = controllerPaths.getPath( + CGroupController.valueOf(controller.toUpperCase())); return controllerPath + "/" + cgroupPrefix + "/" + groupName; } @@ -383,97 +379,24 @@ public String getResourcesOption(ContainerId containerId) { return sb.toString(); } - /* We are looking for entries of the form: - * none /cgroup/path/mem cgroup rw,memory 0 0 - * - * Use a simple pattern that splits on the five spaces, and - * grabs the 2, 3, and 4th fields. - */ - - private static final Pattern MTAB_FILE_FORMAT = Pattern.compile( - "^[^\\s]+\\s([^\\s]+)\\s([^\\s]+)\\s([^\\s]+)\\s[^\\s]+\\s[^\\s]+$"); - - /* - * Returns a map: path -> mount options - * for mounts with type "cgroup". Cgroup controllers will - * appear in the list of options for a path. - */ - private Map> parseMtab() throws IOException { - Map> ret = new HashMap>(); - BufferedReader in = null; - Set validCgroups = - CGroupsHandler.CGroupController.getValidCGroups(); - - try { - FileInputStream fis = new FileInputStream(new File(getMtabFileName())); - in = new BufferedReader(new InputStreamReader(fis, "UTF-8")); - - for (String str = in.readLine(); str != null; - str = in.readLine()) { - Matcher m = MTAB_FILE_FORMAT.matcher(str); - boolean mat = m.find(); - if (mat) { - String path = m.group(1); - String type = m.group(2); - String options = m.group(3); - - if (type.equals(CGROUPS_FSTYPE)) { - Set cgroupList = - new HashSet<>(Arrays.asList(options.split(","))); - // Collect the valid subsystem names - cgroupList.retainAll(validCgroups); - ret.put(path, cgroupList); - } - } - } - } catch (IOException e) { - throw new IOException("Error while reading " + getMtabFileName(), e); - } finally { - IOUtils.cleanupWithLogger(LOG, in); - } - - return ret; - } - - @VisibleForTesting - String findControllerInMtab(String controller, - Map> entries) { - for (Entry> e : entries.entrySet()) { - if (e.getValue().contains(controller)) { - if (new File(e.getKey()).canRead()) { - return e.getKey(); - } else { - LOG.warn(String.format( - "Skipping inaccessible cgroup mount point %s", e.getKey())); - } - } - } - - return null; - } - private void initializeControllerPaths() throws IOException { - String controllerPath; - Map> parsedMtab = null; - + final CGroupsMountConfig mountConfig; if (this.cgroupMountPath != null && !this.cgroupMount) { - parsedMtab = ResourceHandlerModule. + mountConfig = CgroupsFileSystemMountParser. parseConfiguredCGroupPath(this.cgroupMountPath); + } else { + mountConfig = MtabFileParser.parse(getMtabFileName()); } - if (parsedMtab == null) { - parsedMtab = parseMtab(); - } - - // CPU - - controllerPath = findControllerInMtab(CONTROLLER_CPU, parsedMtab); + final String controllerPath = + CGroupsControllerPaths.findControllerPathInMountConfig( + CONTROLLER_CPU, mountConfig); if (controllerPath != null) { File f = new File(controllerPath + "/" + this.cgroupPrefix); if (FileUtil.canWrite(f)) { - controllerPaths.put(CONTROLLER_CPU, controllerPath); + controllerPaths.storePath(CGroupController.CPU, controllerPath); } else { throw new IOException("Not able to enforce cpu weights; cannot write " + "to cgroup at: " + controllerPath); @@ -490,7 +413,7 @@ String getMtabFileName() { } @VisibleForTesting - Map getControllerPaths() { - return Collections.unmodifiableMap(controllerPaths); + CGroupsControllerPaths getControllerPaths() { + return controllerPaths; } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsControllerPaths.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsControllerPaths.java new file mode 100644 index 00000000000..48e553bf1ec --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsControllerPaths.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux + .resources; + +import org.apache.commons.io.FileUtils; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.CGroupsHandler.CGroupController; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.io.File; +import java.util.Collections; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * Tests for {@link CGroupsControllerPaths}. + */ +public class TestCGroupsControllerPaths { + private String tmpPath; + + @Before + public void setup() { + // Prepare test directory + tmpPath = System.getProperty("test.build.data") + "/cgroups"; + File tmpDir = new File(tmpPath); + FileUtils.deleteQuietly(tmpDir); + assertTrue(tmpDir.mkdirs()); + } + + @After + public void teardown() { + FileUtils.deleteQuietly(new File(tmpPath)); + } + + @Test + public void testSelectCgroup() { + File cpu = new File(tmpPath, "cpu"); + File cpuNotExist = new File(tmpPath, "cpuNotExist"); + File memory = new File(tmpPath, "memory"); + try { + assertTrue("temp dir for cpu should be created", cpu.mkdirs()); + assertTrue("temp dir for memory should be created", + memory.mkdirs()); + assertFalse("temp dir for cpu should not be created", + cpuNotExist.exists()); + + CGroupsMountConfig config = new CGroupsMountConfig(); + config.mapPathToControllers(memory.getAbsolutePath(), + Collections.singleton("memory")); + config.mapPathToControllers(cpuNotExist.getAbsolutePath(), + Collections.singleton("cpu")); + config.mapPathToControllers(cpu.getAbsolutePath(), + Collections.singleton("cpu")); + + CGroupsControllerPaths testSubject = CGroupsControllerPaths. + initializeFromMountConfig(config); + String selectedCPU = testSubject.getPath(CGroupController.CPU); + assertEquals("Wrong CPU mount point selected", + cpu.getAbsolutePath(), selectedCPU); + } finally { + FileUtils.deleteQuietly(cpu); + FileUtils.deleteQuietly(memory); + } + } + +} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsHandlerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsHandlerImpl.java index b1e8989213b..80d9fda447d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsHandlerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsHandlerImpl.java @@ -31,7 +31,6 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationException; import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationExecutor; import org.junit.After; -import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.mockito.ArgumentCaptor; @@ -41,13 +40,13 @@ import java.io.IOException; import java.nio.file.Files; import java.security.Permission; -import java.util.Collections; -import java.util.LinkedHashMap; -import java.util.Map; -import java.util.Set; +import java.util.Arrays; import java.util.UUID; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; @@ -169,7 +168,8 @@ public static File createPremountedCgroups(File parentDir, boolean cpuAcct) + " cgroup rw,relatime,blkio 0 0\n"; assertTrue("Directory should be created", blkioCgroup.mkdirs()); - File mockMtab = new File(parentDir, UUID.randomUUID().toString()); + File mockMtab = new File(parentDir, "mockmtab_" + + UUID.randomUUID().toString()); if (!mockMtab.exists()) { if (!mockMtab.createNewFile()) { String message = "Could not create file " + mockMtab.getAbsolutePath(); @@ -189,7 +189,7 @@ public static File createPremountedCgroups(File parentDir, boolean cpuAcct) public void testMountController() throws IOException { File parentDir = new File(tmpPath); File cgroup = new File(parentDir, controller.getName()); - assertTrue("cgroup dir should be cerated", cgroup.mkdirs()); + assertTrue("cgroup dir should be created", cgroup.mkdirs()); //Since we enabled (deferred) cgroup controller mounting, no interactions //should have occurred, with this mock verifyZeroInteractions(privilegedOperationExecutorMock); @@ -217,7 +217,7 @@ public void testMountController() throws IOException { //we'll explicitly capture and assert that the //captured op and the expected op are identical. - Assert.assertEquals(expectedOp, opCaptor.getValue()); + assertEquals(expectedOp, opCaptor.getValue()); verifyNoMoreInteractions(privilegedOperationExecutorMock); //Try mounting the same controller again - this should be a no-op @@ -225,12 +225,11 @@ public void testMountController() throws IOException { verifyNoMoreInteractions(privilegedOperationExecutorMock); } catch (PrivilegedOperationException e) { LOG.error("Caught exception: " + e); - assertTrue("Unexpected PrivilegedOperationException from mock!", - false); + fail("Unexpected PrivilegedOperationException from mock!"); } } catch (ResourceHandlerException e) { LOG.error("Caught exception: " + e); - assertTrue("Unexpected ResourceHandler Exception!", false); + fail("Unexpected ResourceHandler Exception!"); } } @@ -254,27 +253,25 @@ public void testCGroupPaths() throws IOException { cGroupsHandler.initializeCGroupController(controller); } catch (ResourceHandlerException e) { LOG.error("Caught exception: " + e); - assertTrue( - "Unexpected ResourceHandlerException when mounting controller!", - false); + fail("Unexpected ResourceHandlerException when mounting controller!"); } String testCGroup = "container_01"; String expectedPath = controllerPath + Path.SEPARATOR + testCGroup; String path = cGroupsHandler.getPathForCGroup(controller, testCGroup); - Assert.assertEquals(expectedPath, path); + assertEquals(expectedPath, path); String expectedPathTasks = expectedPath + Path.SEPARATOR + CGroupsHandler.CGROUP_PROCS_FILE; path = cGroupsHandler.getPathForCGroupTasks(controller, testCGroup); - Assert.assertEquals(expectedPathTasks, path); + assertEquals(expectedPathTasks, path); String param = CGroupsHandler.CGROUP_PARAM_CLASSID; String expectedPathParam = expectedPath + Path.SEPARATOR + controller.getName() + "." + param; path = cGroupsHandler.getPathForCGroupParam(controller, testCGroup, param); - Assert.assertEquals(expectedPathParam, path); + assertEquals(expectedPathParam, path); } @Test @@ -297,9 +294,7 @@ public void testCGroupOperations() throws IOException { cGroupsHandler.initializeCGroupController(controller); } catch (ResourceHandlerException e) { LOG.error("Caught exception: " + e); - assertTrue( - "Unexpected ResourceHandlerException when mounting controller!", - false); + fail("Unexpected ResourceHandlerException when mounting controller!"); } String testCGroup = "container_01"; @@ -309,7 +304,7 @@ public void testCGroupOperations() throws IOException { String path = cGroupsHandler.createCGroup(controller, testCGroup); assertTrue(new File(expectedPath).exists()); - Assert.assertEquals(expectedPath, path); + assertEquals(expectedPath, path); //update param and read param tests. //We don't use net_cls.classid because as a test param here because @@ -328,14 +323,14 @@ public void testCGroupOperations() throws IOException { assertTrue(paramFile.exists()); try { - Assert.assertEquals(paramValue, new String(Files.readAllBytes( + assertEquals(paramValue, new String(Files.readAllBytes( paramFile.toPath()))); } catch (IOException e) { LOG.error("Caught exception: " + e); - Assert.fail("Unexpected IOException trying to read cgroup param!"); + fail("Unexpected IOException trying to read cgroup param!"); } - Assert.assertEquals(paramValue, + assertEquals(paramValue, cGroupsHandler.getCGroupParam(controller, testCGroup, param)); //We can't really do a delete test here. Linux cgroups @@ -348,42 +343,10 @@ public void testCGroupOperations() throws IOException { //delete is not possible with a regular non-empty directory. } catch (ResourceHandlerException e) { LOG.error("Caught exception: " + e); - Assert - .fail("Unexpected ResourceHandlerException during cgroup operations!"); + fail("Unexpected ResourceHandlerException during cgroup operations!"); } } - /** - * Tests whether mtab parsing works as expected with a valid hierarchy set. - * @throws Exception the test will fail - */ - @Test - public void testMtabParsing() throws Exception { - // Initialize mtab and cgroup dir - File parentDir = new File(tmpPath); - // create mock cgroup - File mockMtabFile = createPremountedCgroups(parentDir, false); - - // Run mtabs parsing - Map> newMtab = - CGroupsHandlerImpl.parseMtab(mockMtabFile.getAbsolutePath()); - Map controllerPaths = - CGroupsHandlerImpl.initializeControllerPathsFromMtab( - newMtab); - - // Verify - Assert.assertEquals(2, controllerPaths.size()); - assertTrue(controllerPaths - .containsKey(CGroupsHandler.CGroupController.CPU)); - assertTrue(controllerPaths - .containsKey(CGroupsHandler.CGroupController.BLKIO)); - String cpuDir = controllerPaths.get(CGroupsHandler.CGroupController.CPU); - String blkioDir = - controllerPaths.get(CGroupsHandler.CGroupController.BLKIO); - Assert.assertEquals(parentDir.getAbsolutePath() + "/cpu", cpuDir); - Assert.assertEquals(parentDir.getAbsolutePath() + "/blkio", blkioDir); - } - /** * Tests whether mtab parsing works as expected with the specified hierarchy. * @param myHierarchy path to local cgroup hierarchy @@ -396,11 +359,17 @@ private void testPreMountedControllerInitialization(String myHierarchy) File mtab = createPremountedCgroups(parentDir, false); File mountPoint = new File(parentDir, "cpu"); + LOG.info("Listing of cgroups dir: " + Arrays.toString( + parentDir.listFiles())); + // Initialize YARN classes Configuration confNoMount = createNoMountConfiguration(myHierarchy); CGroupsHandlerImpl cGroupsHandler = new CGroupsHandlerImpl(confNoMount, privilegedOperationExecutorMock, mtab.getAbsolutePath()); + LOG.info("Recognized cgroups controller paths: " + + cGroupsHandler.getControllerPaths()); + File cpuCgroupMountDir = new File( cGroupsHandler.getPathForCGroup(CGroupsHandler.CGroupController.CPU, "")); @@ -419,7 +388,7 @@ private void testPreMountedControllerInitialization(String myHierarchy) try { cGroupsHandler.initializeCGroupController( CGroupsHandler.CGroupController.CPU); - Assert.fail("An inaccessible path should result in an exception"); + fail("An inaccessible path should result in an exception"); } catch (Exception e) { assertTrue("Unexpected exception " + e.getClass().toString(), e instanceof ResourceHandlerException); @@ -438,7 +407,7 @@ private void testPreMountedControllerInitialization(String myHierarchy) try { cGroupsHandler.initializeCGroupController( CGroupsHandler.CGroupController.CPU); - Assert.fail("An inaccessible path should result in an exception"); + fail("An inaccessible path should result in an exception"); } catch (Exception e) { assertTrue("Unexpected exception " + e.getClass().toString(), e instanceof ResourceHandlerException); @@ -449,7 +418,7 @@ private void testPreMountedControllerInitialization(String myHierarchy) // Test that a SecurityException results in an exception if (!cpuCgroupMountDir.equals(mountPoint)) { - Assert.assertFalse("Could not delete cgroups", + assertFalse("Could not delete cgroups", cpuCgroupMountDir.delete()); assertTrue("Directory should be deleted", !cpuCgroupMountDir.exists()); @@ -458,7 +427,7 @@ private void testPreMountedControllerInitialization(String myHierarchy) try { cGroupsHandler.initializeCGroupController( CGroupsHandler.CGroupController.CPU); - Assert.fail("An inaccessible path should result in an exception"); + fail("An inaccessible path should result in an exception"); } catch (Exception e) { assertTrue("Unexpected exception " + e.getClass().toString(), e instanceof ResourceHandlerException); @@ -469,7 +438,7 @@ private void testPreMountedControllerInitialization(String myHierarchy) // Test that a non-existing mount directory results in an exception if (!cpuCgroupMountDir.equals(mountPoint)) { - Assert.assertFalse("Could not delete cgroups", + assertFalse("Could not delete cgroups", cpuCgroupMountDir.delete()); assertTrue("Directory should be deleted", !cpuCgroupMountDir.exists()); @@ -480,42 +449,13 @@ private void testPreMountedControllerInitialization(String myHierarchy) try { cGroupsHandler.initializeCGroupController( CGroupsHandler.CGroupController.CPU); - Assert.fail("An inaccessible path should result in an exception"); + fail("An inaccessible path should result in an exception"); } catch (Exception e) { assertTrue("Unexpected exception " + e.getClass().toString(), e instanceof ResourceHandlerException); } } - @Test - public void testSelectCgroup() throws Exception { - File cpu = new File(tmpPath, "cpu"); - File cpuNoExist = new File(tmpPath, "cpuNoExist"); - File memory = new File(tmpPath, "memory"); - try { - CGroupsHandlerImpl handler = new CGroupsHandlerImpl( - createNoMountConfiguration(tmpPath), - privilegedOperationExecutorMock); - Map> cgroups = new LinkedHashMap<>(); - - Assert.assertTrue("temp dir should be created", cpu.mkdirs()); - Assert.assertTrue("temp dir should be created", memory.mkdirs()); - Assert.assertFalse("temp dir should not be created", cpuNoExist.exists()); - - cgroups.put( - memory.getAbsolutePath(), Collections.singleton("memory")); - cgroups.put( - cpuNoExist.getAbsolutePath(), Collections.singleton("cpu")); - cgroups.put(cpu.getAbsolutePath(), Collections.singleton("cpu")); - String selectedCPU = handler.findControllerInMtab("cpu", cgroups); - Assert.assertEquals("Wrong CPU mount point selected", - cpu.getAbsolutePath(), selectedCPU); - } finally { - FileUtils.deleteQuietly(cpu); - FileUtils.deleteQuietly(memory); - } - } - /** * Tests whether mtab parsing works as expected with an empty hierarchy set. * @throws Exception the test will fail @@ -584,13 +524,13 @@ public void testManualCgroupSetting() throws ResourceHandlerException { File cpu = new File(new File(tmpPath, "cpuacct,cpu"), "/hadoop-yarn"); try { - Assert.assertTrue("temp dir should be created", cpu.mkdirs()); + assertTrue("temp dir should be created", cpu.mkdirs()); CGroupsHandlerImpl cGroupsHandler = new CGroupsHandlerImpl(conf, null); cGroupsHandler.initializeCGroupController( CGroupsHandler.CGroupController.CPU); - Assert.assertEquals("CPU CGRoup path was not set", cpu.getAbsolutePath(), + assertEquals("CPU cgroup path was not set", cpu.getAbsolutePath(), new File(cGroupsHandler.getPathForCGroup( CGroupsHandler.CGroupController.CPU, "")).getAbsolutePath()); @@ -608,32 +548,32 @@ public void testCgroupsHierarchySetting() throws ResourceHandlerException { "/hadoop-yarn"); CGroupsHandlerImpl cGroupsHandler = new CGroupsHandlerImpl(conf, null); String expectedRelativePath = "hadoop-yarn/c1"; - Assert.assertEquals(expectedRelativePath, + assertEquals(expectedRelativePath, cGroupsHandler.getRelativePathForCGroup("c1")); conf.set(YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_HIERARCHY, "hadoop-yarn"); cGroupsHandler = new CGroupsHandlerImpl(conf, null); - Assert.assertEquals(expectedRelativePath, + assertEquals(expectedRelativePath, cGroupsHandler.getRelativePathForCGroup("c1")); conf.set(YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_HIERARCHY, "hadoop-yarn/"); cGroupsHandler = new CGroupsHandlerImpl(conf, null); - Assert.assertEquals(expectedRelativePath, + assertEquals(expectedRelativePath, cGroupsHandler.getRelativePathForCGroup("c1")); conf.set(YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_HIERARCHY, "//hadoop-yarn//"); cGroupsHandler = new CGroupsHandlerImpl(conf, null); - Assert.assertEquals(expectedRelativePath, + assertEquals(expectedRelativePath, cGroupsHandler.getRelativePathForCGroup("c1")); expectedRelativePath = "hadoop-yarn/root/c1"; conf.set(YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_HIERARCHY, "//hadoop-yarn/root//"); cGroupsHandler = new CGroupsHandlerImpl(conf, null); - Assert.assertEquals(expectedRelativePath, + assertEquals(expectedRelativePath, cGroupsHandler.getRelativePathForCGroup("c1")); } } \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestMtabFileParser.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestMtabFileParser.java new file mode 100644 index 00000000000..79261bf1546 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestMtabFileParser.java @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources; + +import org.apache.commons.io.FileUtils; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.util.Arrays; + +import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.TestCGroupsHandlerImpl.createPremountedCgroups; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * Tests for {@link MtabFileParser}. + */ +public class TestMtabFileParser { + private static final Logger LOG = + LoggerFactory.getLogger(TestMtabFileParser.class); + + private String tmpPath; + + @Before + public void setup() { + // Prepare test directory + tmpPath = System.getProperty("test.build.data") + "/cgroups"; + File tmpDir = new File(tmpPath); + FileUtils.deleteQuietly(tmpDir); + assertTrue(tmpDir.mkdirs()); + } + + @After + public void teardown() { + FileUtils.deleteQuietly(new File(tmpPath)); + } + + /** + * Tests whether mtab parsing works as expected with a valid hierarchy set. + * @throws Exception the test will fail + */ + @Test + public void testMtabParsing() throws Exception { + // Initialize mtab and cgroup dir + File parentDir = new File(tmpPath); + // create mock cgroup + File mockMtabFile = createPremountedCgroups(parentDir, false); + + // Run mtabs parsing + CGroupsMountConfig mountConfig = + MtabFileParser.parse(mockMtabFile.getAbsolutePath()); + CGroupsControllerPaths controllerPaths = + CGroupsControllerPaths.initializeFromMountConfig(mountConfig); + + LOG.info("Listing of cgroups dir: " + + Arrays.toString(parentDir.listFiles())); + LOG.info("Recognized cgroups controller paths: " + controllerPaths); + // Verify + assertEquals("Number of cgroups controller paths is not expected", + 2, controllerPaths.getNumberOfPaths()); + assertTrue("There is no path for CPU in controllerPaths", controllerPaths + .hasPath(CGroupsHandler.CGroupController.CPU)); + assertTrue("There is no path for BLKIO in controllerPaths", controllerPaths + .hasPath(CGroupsHandler.CGroupController.BLKIO)); + String cpuDir = controllerPaths.getPath( + CGroupsHandler.CGroupController.CPU); + String blkioDir = + controllerPaths.getPath(CGroupsHandler.CGroupController.BLKIO); + assertEquals(parentDir.getAbsolutePath() + "/cpu", cpuDir); + assertEquals(parentDir.getAbsolutePath() + "/blkio", blkioDir); + } +} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestCgroupsLCEResourcesHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestCgroupsLCEResourcesHandler.java index 7d8704f8b42..0987ab6e509 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestCgroupsLCEResourcesHandler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestCgroupsLCEResourcesHandler.java @@ -20,27 +20,29 @@ import org.apache.commons.io.FileUtils; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.nodemanager.LinuxContainerExecutor; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.CGroupsHandler.CGroupController; import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.TestCGroupsHandlerImpl; - import org.apache.hadoop.yarn.util.ControlledClock; import org.apache.hadoop.yarn.util.ResourceCalculatorPlugin; -import org.junit.Assert; -import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.junit.Test; import org.junit.After; import org.junit.Before; +import org.junit.Test; import org.mockito.Mockito; -import java.io.*; -import java.util.Collections; -import java.util.LinkedHashMap; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; import java.util.List; -import java.util.Map; import java.util.Scanner; -import java.util.Set; import java.util.concurrent.CountDownLatch; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; import static org.mockito.Mockito.when; @Deprecated @@ -70,7 +72,7 @@ public void testcheckAndDeleteCgroup() throws Exception { FileUtils.deleteQuietly(cgroupDir); // Test 0 // tasks file not present, should return false - Assert.assertFalse(handler.checkAndDeleteCgroup(cgroupDir)); + assertFalse(handler.checkAndDeleteCgroup(cgroupDir)); File tfile = new File(cgroupDir.getAbsolutePath(), "tasks"); FileOutputStream fos = FileUtils.openOutputStream(tfile); @@ -79,13 +81,13 @@ public void testcheckAndDeleteCgroup() throws Exception { // Test 1, tasks file is empty // tasks file has no data, should return true Mockito.when(fspy.delete()).thenReturn(true); - Assert.assertTrue(handler.checkAndDeleteCgroup(fspy)); + assertTrue(handler.checkAndDeleteCgroup(fspy)); // Test 2, tasks file has data fos.write("1234".getBytes()); fos.close(); // tasks has data, would not be able to delete, should return false - Assert.assertFalse(handler.checkAndDeleteCgroup(fspy)); + assertFalse(handler.checkAndDeleteCgroup(fspy)); FileUtils.deleteQuietly(cgroupDir); } @@ -122,7 +124,7 @@ public void run() { } }.start(); latch.await(); - Assert.assertFalse(handler.deleteCgroup(cgroupDir.getAbsolutePath())); + assertFalse(handler.deleteCgroup(cgroupDir.getAbsolutePath())); FileUtils.deleteQuietly(cgroupDir); } @@ -202,8 +204,8 @@ public void testInit() throws IOException { handler.init(mockLCE, plugin); File periodFile = new File(cpuCgroupMountDir, "cpu.cfs_period_us"); File quotaFile = new File(cpuCgroupMountDir, "cpu.cfs_quota_us"); - Assert.assertFalse(periodFile.exists()); - Assert.assertFalse(quotaFile.exists()); + assertFalse(periodFile.exists()); + assertFalse(quotaFile.exists()); // subset of cpu being used, files should be created conf @@ -213,8 +215,8 @@ public void testInit() throws IOException { handler.init(mockLCE, plugin); int period = readIntFromFile(periodFile); int quota = readIntFromFile(quotaFile); - Assert.assertEquals(100 * 1000, period); - Assert.assertEquals(1000 * 1000, quota); + assertEquals(100 * 1000, period); + assertEquals(1000 * 1000, quota); // set cpu back to 100, quota should be -1 conf.setInt(YarnConfiguration.NM_RESOURCE_PERCENTAGE_PHYSICAL_CPU_LIMIT, @@ -223,7 +225,7 @@ public void testInit() throws IOException { handler.limits[1] = 1000 * 1000; handler.init(mockLCE, plugin); quota = readIntFromFile(quotaFile); - Assert.assertEquals(-1, quota); + assertEquals(-1, quota); FileUtils.deleteQuietly(cgroupDir); } @@ -244,18 +246,18 @@ public void testGetOverallLimits() { CgroupsLCEResourcesHandler handler = new CgroupsLCEResourcesHandler(); int[] ret = handler.getOverallLimits(2); - Assert.assertEquals(expectedQuota / 2, ret[0]); - Assert.assertEquals(expectedQuota, ret[1]); + assertEquals(expectedQuota / 2, ret[0]); + assertEquals(expectedQuota, ret[1]); ret = handler.getOverallLimits(2000); - Assert.assertEquals(expectedQuota, ret[0]); - Assert.assertEquals(-1, ret[1]); + assertEquals(expectedQuota, ret[0]); + assertEquals(-1, ret[1]); int[] params = {0, -1}; for (int cores : params) { try { handler.getOverallLimits(cores); - Assert.fail("Function call should throw error."); + fail("Function call should throw error."); } catch (IllegalArgumentException ie) { // expected } @@ -263,8 +265,8 @@ public void testGetOverallLimits() { // test minimums ret = handler.getOverallLimits(1000 * 1000); - Assert.assertEquals(1000 * 1000, ret[0]); - Assert.assertEquals(-1, ret[1]); + assertEquals(1000 * 1000, ret[0]); + assertEquals(-1, ret[1]); } @Test @@ -298,16 +300,17 @@ public void testContainerLimits() throws IOException { // check the controller paths map isn't empty ContainerId id = ContainerId.fromString("container_1_1_1_1"); handler.preExecute(id, Resource.newInstance(1024, 1)); - Assert.assertNotNull(handler.getControllerPaths()); + assertNotNull(handler.getControllerPaths()); + assertFalse(handler.getControllerPaths().isEmpty()); // check values // default case - files shouldn't exist, strict mode off by default File containerCpuDir = new File(cpuCgroupMountDir, id.toString()); - Assert.assertTrue(containerCpuDir.exists()); - Assert.assertTrue(containerCpuDir.isDirectory()); + assertTrue(containerCpuDir.exists()); + assertTrue(containerCpuDir.isDirectory()); File periodFile = new File(containerCpuDir, "cpu.cfs_period_us"); File quotaFile = new File(containerCpuDir, "cpu.cfs_quota_us"); - Assert.assertFalse(periodFile.exists()); - Assert.assertFalse(quotaFile.exists()); + assertFalse(periodFile.exists()); + assertFalse(quotaFile.exists()); // no files created because we're using all cpu FileUtils.deleteQuietly(containerCpuDir); @@ -317,12 +320,12 @@ public void testContainerLimits() throws IOException { handler.initConfig(); handler.preExecute(id, Resource.newInstance(1024, YarnConfiguration.DEFAULT_NM_VCORES)); - Assert.assertTrue(containerCpuDir.exists()); - Assert.assertTrue(containerCpuDir.isDirectory()); + assertTrue(containerCpuDir.exists()); + assertTrue(containerCpuDir.isDirectory()); periodFile = new File(containerCpuDir, "cpu.cfs_period_us"); quotaFile = new File(containerCpuDir, "cpu.cfs_quota_us"); - Assert.assertFalse(periodFile.exists()); - Assert.assertFalse(quotaFile.exists()); + assertFalse(periodFile.exists()); + assertFalse(quotaFile.exists()); // 50% of CPU FileUtils.deleteQuietly(containerCpuDir); @@ -332,14 +335,14 @@ public void testContainerLimits() throws IOException { handler.initConfig(); handler.preExecute(id, Resource.newInstance(1024, YarnConfiguration.DEFAULT_NM_VCORES / 2)); - Assert.assertTrue(containerCpuDir.exists()); - Assert.assertTrue(containerCpuDir.isDirectory()); + assertTrue(containerCpuDir.exists()); + assertTrue(containerCpuDir.isDirectory()); periodFile = new File(containerCpuDir, "cpu.cfs_period_us"); quotaFile = new File(containerCpuDir, "cpu.cfs_quota_us"); - Assert.assertTrue(periodFile.exists()); - Assert.assertTrue(quotaFile.exists()); - Assert.assertEquals(500 * 1000, readIntFromFile(periodFile)); - Assert.assertEquals(1000 * 1000, readIntFromFile(quotaFile)); + assertTrue(periodFile.exists()); + assertTrue(quotaFile.exists()); + assertEquals(500 * 1000, readIntFromFile(periodFile)); + assertEquals(1000 * 1000, readIntFromFile(quotaFile)); // CGroups set to 50% of CPU, container set to 50% of YARN CPU FileUtils.deleteQuietly(containerCpuDir); @@ -352,45 +355,18 @@ public void testContainerLimits() throws IOException { handler.init(mockLCE, plugin); handler.preExecute(id, Resource.newInstance(1024, YarnConfiguration.DEFAULT_NM_VCORES / 2)); - Assert.assertTrue(containerCpuDir.exists()); - Assert.assertTrue(containerCpuDir.isDirectory()); + assertTrue(containerCpuDir.exists()); + assertTrue(containerCpuDir.isDirectory()); periodFile = new File(containerCpuDir, "cpu.cfs_period_us"); quotaFile = new File(containerCpuDir, "cpu.cfs_quota_us"); - Assert.assertTrue(periodFile.exists()); - Assert.assertTrue(quotaFile.exists()); - Assert.assertEquals(1000 * 1000, readIntFromFile(periodFile)); - Assert.assertEquals(1000 * 1000, readIntFromFile(quotaFile)); + assertTrue(periodFile.exists()); + assertTrue(quotaFile.exists()); + assertEquals(1000 * 1000, readIntFromFile(periodFile)); + assertEquals(1000 * 1000, readIntFromFile(quotaFile)); FileUtils.deleteQuietly(cgroupDir); } - @Test - public void testSelectCgroup() { - File cpu = new File(cgroupDir, "cpu"); - File cpuNoExist = new File(cgroupDir, "cpuNoExist"); - File memory = new File(cgroupDir, "memory"); - try { - CgroupsLCEResourcesHandler handler = new CgroupsLCEResourcesHandler(); - Map> cgroups = new LinkedHashMap<>(); - - Assert.assertTrue("temp dir should be created", cpu.mkdirs()); - Assert.assertTrue("temp dir should be created", memory.mkdirs()); - Assert.assertFalse("temp dir should not be created", cpuNoExist.exists()); - - cgroups.put( - memory.getAbsolutePath(), Collections.singleton("memory")); - cgroups.put( - cpuNoExist.getAbsolutePath(), Collections.singleton("cpu")); - cgroups.put(cpu.getAbsolutePath(), Collections.singleton("cpu")); - String selectedCPU = handler.findControllerInMtab("cpu", cgroups); - Assert.assertEquals("Wrong CPU mount point selected", - cpu.getAbsolutePath(), selectedCPU); - } finally { - FileUtils.deleteQuietly(cpu); - FileUtils.deleteQuietly(memory); - } - } - @Test public void testManualCgroupSetting() throws IOException { CgroupsLCEResourcesHandler handler = new CgroupsLCEResourcesHandler(); @@ -401,7 +377,7 @@ public void testManualCgroupSetting() throws IOException { File cpu = new File(new File(cgroupDir, "cpuacct,cpu"), "/hadoop-yarn"); try { - Assert.assertTrue("temp dir should be created", cpu.mkdirs()); + assertTrue("temp dir should be created", cpu.mkdirs()); final int numProcessors = 4; ResourceCalculatorPlugin plugin = @@ -411,8 +387,8 @@ public void testManualCgroupSetting() throws IOException { when(plugin.getNumProcessors()).thenReturn(8); handler.init(null, plugin); - Assert.assertEquals("CPU CGRoup path was not set", cpu.getParent(), - handler.getControllerPaths().get("cpu")); + assertEquals("CPU CGRoup path was not set", cpu.getParent(), + handler.getControllerPaths().getPath(CGroupController.CPU)); } finally { FileUtils.deleteQuietly(cpu);