diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java index b2dfb78..d035a8c 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java @@ -89,10 +89,10 @@ // to enable testing, using this variable which can be configured // to a test directory. - private String procfsDir; + protected String procfsDir; static private String deadPid = "-1"; - private String pid = deadPid; + protected String pid = deadPid; static private Pattern numberPattern = Pattern.compile("[1-9][0-9]*"); private Long cpuTime = 0L; @@ -331,7 +331,7 @@ public long getCumulativeCpuTime() { return cpuTime; } - private static String getValidPID(String pid) { + static String getValidPID(String pid) { if (pid == null) return deadPid; Matcher m = numberPattern.matcher(pid); if (m.matches()) return pid; @@ -439,7 +439,7 @@ public String toString() { * Class containing information of a process. * */ - private static class ProcessInfo { + static class ProcessInfo { private String pid; // process-id private String name; // command name private Integer pgrpId; // process group-id diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/SmapsBasedProcessTree.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/SmapsBasedProcessTree.java new file mode 100644 index 0000000..5331f24 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/SmapsBasedProcessTree.java @@ -0,0 +1,406 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.util; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.FileReader; +import java.io.IOException; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.apache.commons.io.IOUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +/** + * + * SMAPs (/proc/pid/smaps) based ProcessTree. Works only on Linux. + * + */ +public class SmapsBasedProcessTree extends ProcfsBasedProcessTree { + static final Log LOG = LogFactory.getLog(SmapsBasedProcessTree.class); + + private static final String PROCFS = "/proc/"; + + enum MEM_INFO { + SIZE("Size"), RSS("Rss"), PSS("Pss"), SHARED_CLEAN("Shared_Clean"), SHARED_DIRTY( + "Shared_Dirty"), PRIVATE_CLEAN("Private_Clean"), PRIVATE_DIRTY( + "Private_Dirty"), REFERENCED("Referenced"), ANONYMOUS("Anonymous"), ANON_HUGE_PAGES( + "AnonHugePages"), SWAP("swap"), KERNEL_PAGE_SIZE("kernelPageSize"), MMU_PAGE_SIZE( + "mmuPageSize"), INVALID("invalid"); + + private String name; + + private MEM_INFO(String name) { + this.name = name; + } + + public static MEM_INFO getMemInfoByName(String name) { + for (MEM_INFO info : MEM_INFO.values()) { + if (info.name.trim().equals(name.trim())) { + return info; + } + } + return INVALID; + } + } + + public static final String SMAPS = "smaps"; + private static final String KB = "kB"; + private static final String READ_ONLY_WITH_SHARED_PERMISSION = "r--s"; + private static final String READ_EXECUTE_WITH_SHARED_PERMISSION = "r-xs"; + + private static final Pattern ADDRESS_PATTERN = Pattern + .compile("([[a-f]|(0-9)]*)-([[a-f]|(0-9)]*)(\\s)*([rxwps\\-]*)"); + private static final Pattern MEM_INFO_PATTERN = Pattern + .compile("(^[A-Z].*):[\t ]+(.*)"); + + protected Map processSMAPTree = new HashMap(); + + public SmapsBasedProcessTree(String pid) { + this(pid, PROCFS); + } + + /** + * Build a new process tree rooted at the pid. + * + * This method is provided mainly for testing purposes, where the root of the + * proc file system can be adjusted. + * + * @param pid + * root of the process tree + * @param procfsDir + * the root of a proc file system - only used for testing. + */ + public SmapsBasedProcessTree(String pid, String procfsDir) { + super(pid); + this.pid = getValidPID(pid); + this.procfsDir = procfsDir; + } + + /** + * Update process-tree with latest state. If the root-process is not alive, + * tree will be empty. + * + */ + @Override + public void updateProcessTree() { + super.updateProcessTree(); // Update PSS related information + for (ProcessInfo p : processTree.values()) { + if (p != null) { + // Get information for each process + ProcessMemInfo memInfo = new ProcessMemInfo(p.getPid()); + constructProcessSMAPInfo(memInfo, procfsDir); + processSMAPTree.put(p.getPid(), memInfo); + } + } + } + + /** + * Get the cumulative resident set size (RSS) memory used by all the processes + * in the process-tree that are older than the passed in age. RSS is + * calculated based on SMAP information. Skip mappings with "r--s", "r-xs" + * permissions to get real RSS usage of the process. + * + * + * @param olderThanAge + * processes above this age are included in the memory addition + * @return cumulative rss memory used by the process-tree in bytes, for + * processes older than this age. return 0 if it cannot be calculated + */ + @Override + public long getCumulativeRssmem(int olderThanAge) { + long total = 0; + for (ProcessInfo p : processTree.values()) { + if ((p != null) && (p.getAge() > olderThanAge)) { + ProcessMemInfo procMemInfo = processSMAPTree.get(p.getPid()); + if (procMemInfo != null) { + for (MemoryMappingInfo info : procMemInfo.getModuleMemList()) { + // Do not account for r--s or r-xs mappings + if (info.getPermission().trim() + .equalsIgnoreCase(READ_ONLY_WITH_SHARED_PERMISSION) || + info.getPermission().trim() + .equalsIgnoreCase(READ_EXECUTE_WITH_SHARED_PERMISSION)) { + continue; + } + total += Math.min(info.sharedDirty, info.pss) + info.privateDirty + + info.privateClean; + if (LOG.isDebugEnabled()) { + LOG.debug(" total(" + olderThanAge + "): PID : " + p.getPid() + + ", SharedDirty : " + info.sharedDirty + ", PSS : " + + info.pss + ", Private_Dirty : " + info.privateDirty + + ", Private_Clean : " + info.privateClean + ", total : " + + (total * 1024)); + } + } + } + if (LOG.isDebugEnabled()) { + LOG.debug(procMemInfo.toString()); + } + } + } + total = (total * 1024); // convert to bytes + return total; // size + } + + /** + * Update memory related information + * + * @param pinfo + * @param procfsDir + */ + private static void constructProcessSMAPInfo(ProcessMemInfo pInfo, + String procfsDir) { + BufferedReader in = null; + FileReader fReader = null; + try { + File pidDir = new File(procfsDir, pInfo.getPid()); + File file = new File(pidDir, SMAPS); + fReader = new FileReader(file); + in = new BufferedReader(fReader); + MemoryMappingInfo memoryMappingInfo = null; + List lines = IOUtils.readLines(new FileInputStream(file)); + for (String line : lines) { + line = line.trim(); + try { + Matcher address = ADDRESS_PATTERN.matcher(line); + if (address.find()) { + memoryMappingInfo = new MemoryMappingInfo(line); + memoryMappingInfo.setPermission(address.group(4)); + pInfo.getModuleMemList().add(memoryMappingInfo); + continue; + } + Matcher memInfo = MEM_INFO_PATTERN.matcher(line); + if (memInfo.find()) { + String key = memInfo.group(1).trim(); + String value = memInfo.group(2).replace(KB, "").trim(); + if (LOG.isDebugEnabled()) { + LOG.debug("MemInfo : " + key + " : Value : " + value); + } + memoryMappingInfo.updateModuleMemInfo(key, value); + } + } catch (Throwable t) { + LOG.warn("Error parsing smaps line : " + line + "; " + t.getMessage()); + } + } + } catch (FileNotFoundException f) { + LOG.error(f.getMessage()); + } catch (IOException e) { + LOG.error(e.getMessage()); + } catch (Throwable t) { + LOG.error(t.getMessage()); + } finally { + IOUtils.closeQuietly(in); + } + } + + /** + * Placeholder for process's SMAPS information + */ + static class ProcessMemInfo { + private String pid; + private List moduleMemList; + + public ProcessMemInfo(String pid) { + this.pid = pid; + this.moduleMemList = new LinkedList(); + } + + public List getModuleMemList() { + return moduleMemList; + } + + public String getPid() { + return pid; + } + + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("pid : " + pid); + for (MemoryMappingInfo info : moduleMemList) { + sb.append("\n"); + sb.append(info.toString()); + } + return sb.toString(); + } + } + + /** + *
+   * Private Pages : Pages that were mapped only by the process
+   * Shared Pages : Pages that were shared with other processes
+   * 
+   * Clean Pages : Pages that have not been modified since they were mapped
+   * Dirty Pages : Pages that have been modified since they were mapped
+   * 
+   * Private RSS = Private Clean Pages + Private Dirty Pages
+   * Shared RSS = Shared Clean Pages + Shared Dirty Pages
+   * RSS = Private RSS + Shared RSS
+   * PSS = The count of all pages mapped uniquely by the process, 
+   *  plus a fraction of each shared page, said fraction to be 
+   *  proportional to the number of processes which have mapped the page.
+   * 
+   * 
+ */ + static class MemoryMappingInfo { + private int size; + private int rss; + private int pss; + private int sharedClean; + private int sharedDirty; + private int privateClean; + private int privateDirty; + private int referenced; + private String regionName; + private String permission; + + public MemoryMappingInfo(String name) { + this.regionName = name; + } + + public String getName() { + return regionName; + } + + public void setPermission(String permission) { + this.permission = permission; + } + + public String getPermission() { + return permission; + } + + public int getSize() { + return size; + } + + public int getRss() { + return rss; + } + + public int getPss() { + return pss; + } + + public int getSharedClean() { + return sharedClean; + } + + public int getSharedDirty() { + return sharedDirty; + } + + public int getPrivateClean() { + return privateClean; + } + + public int getPrivateDirty() { + return privateDirty; + } + + public int getReferenced() { + return referenced; + } + + public void updateModuleMemInfo(String key, String value) { + MEM_INFO info = MEM_INFO.getMemInfoByName(key); + int val = 0; + try { + val = Integer.parseInt(value.trim()); + } catch (NumberFormatException ne) { + LOG.error("Error in parsing : " + info + " : value" + value.trim()); + } + if (info == null) { + return; + } + if (LOG.isDebugEnabled()) { + LOG.debug("updateModuleMemInfo : memInfo : " + info); + } + switch (info) { + case SIZE: + size = val; + break; + case RSS: + rss = val; + break; + case PSS: + pss = val; + break; + case SHARED_CLEAN: + sharedClean = val; + break; + case SHARED_DIRTY: + sharedDirty = val; + break; + case PRIVATE_CLEAN: + privateClean = val; + break; + case PRIVATE_DIRTY: + privateDirty = val; + break; + case REFERENCED: + referenced = val; + break; + default: + break; + } + } + + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("\t").append(this.getName()).append("\n"); + sb.append("\t").append(this.regionName).append("\n"); + sb.append("\t").append(MEM_INFO.SIZE + ":" + this.getSize()) + .append(" kB\n"); + sb.append("\t").append(MEM_INFO.PSS + ":" + this.getPss()) + .append(" kB\n"); + sb.append("\t").append(MEM_INFO.RSS + ":" + this.getRss()) + .append(" kB\n"); + sb.append("\t") + .append(MEM_INFO.SHARED_CLEAN + ":" + this.getSharedClean()) + .append(" kB\n"); + sb.append("\t") + .append(MEM_INFO.SHARED_DIRTY + ":" + this.getSharedDirty()) + .append(" kB\n"); + sb.append("\t") + .append(MEM_INFO.PRIVATE_CLEAN + ":" + this.getPrivateClean()) + .append(" kB\n"); + sb.append("\t") + .append(MEM_INFO.PRIVATE_DIRTY + ":" + this.getPrivateDirty()) + .append(" kB\n"); + sb.append("\t").append(MEM_INFO.REFERENCED + ":" + this.getReferenced()) + .append(" kB\n"); + sb.append("\t") + .append(MEM_INFO.PRIVATE_DIRTY + ":" + this.getPrivateDirty()) + .append(" kB\n"); + sb.append("\t") + .append(MEM_INFO.PRIVATE_DIRTY + ":" + this.getPrivateDirty()) + .append(" kB\n"); + + return sb.toString(); + } + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestSmapsBasedProcessTree.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestSmapsBasedProcessTree.java new file mode 100755 index 0000000..bb0ab67 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestSmapsBasedProcessTree.java @@ -0,0 +1,962 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.util; + +import static org.junit.Assert.fail; + +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.IOException; +import java.util.LinkedList; +import java.util.List; +import java.util.Random; +import java.util.Vector; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.fs.FileContext; +import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.util.StringUtils; +import org.apache.hadoop.util.Shell; +import org.apache.hadoop.util.Shell.ExitCodeException; +import org.apache.hadoop.util.Shell.ShellCommandExecutor; +import org.apache.hadoop.yarn.util.ProcfsBasedProcessTree; +import org.apache.hadoop.yarn.util.SmapsBasedProcessTree; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +/** + * Similar to ProcfsBasedProcessTree test case. + * Provides add-on checks for RSS memory. + */ +public class TestSmapsBasedProcessTree { + + private static final Log LOG = LogFactory + .getLog(TestSmapsBasedProcessTree.class); + protected static File TEST_ROOT_DIR = new File("target", + TestSmapsBasedProcessTree.class.getName() + "-localDir"); + + private ShellCommandExecutor shexec = null; + private String pidFile, lowestDescendant; + private String shellScript; + private String LINE_SEPARATOR = System.getProperty("line.separator"); + private static final int N = 6; // Controls the RogueTask + + private MemoryMappingInfo[] memInfo = new MemoryMappingInfo[4]; + + private class RogueTaskThread extends Thread { + public void run() { + try { + Vector args = new Vector(); + if(isSetsidAvailable()) { + args.add("setsid"); + } + args.add("bash"); + args.add("-c"); + args.add(" echo $$ > " + pidFile + "; sh " + + shellScript + " " + N + ";") ; + shexec = new ShellCommandExecutor(args.toArray(new String[0])); + shexec.execute(); + } catch (ExitCodeException ee) { + LOG.info("Shell Command exit with a non-zero exit code. This is" + + " expected as we are killing the subprocesses of the" + + " task intentionally. " + ee); + } catch (IOException ioe) { + LOG.info("Error executing shell command " + ioe); + } finally { + LOG.info("Exit code: " + shexec.getExitCode()); + } + } + } + + private String getRogueTaskPID() { + File f = new File(pidFile); + while (!f.exists()) { + try { + Thread.sleep(500); + } catch (InterruptedException ie) { + break; + } + } + + // read from pidFile + return getPidFromPidFile(pidFile); + } + + @Before + public void setup() throws IOException { + FileContext.getLocalFSFileContext().delete( + new Path(TEST_ROOT_DIR.getAbsolutePath()), true); + } + + @Test (timeout = 30000) + public void testProcessTree() throws Exception { + + if (!Shell.LINUX) { + System.out + .println("SmapsBasedProcessTree is not available on this system. Not testing"); + return; + + } + try { + Assert.assertTrue(SmapsBasedProcessTree.isAvailable()); + } catch (Exception e) { + LOG.info(StringUtils.stringifyException(e)); + Assert.assertTrue("SmapsBasedProcessTree should be available on Linux", + false); + return; + } + // create shell script + Random rm = new Random(); + File tempFile = + new File(TEST_ROOT_DIR, getClass().getName() + "_shellScript_" + + rm.nextInt() + ".sh"); + tempFile.deleteOnExit(); + shellScript = TEST_ROOT_DIR + File.separator + tempFile.getName(); + + // create pid file + tempFile = + new File(TEST_ROOT_DIR, getClass().getName() + "_pidFile_" + + rm.nextInt() + ".pid"); + tempFile.deleteOnExit(); + pidFile = TEST_ROOT_DIR + File.separator + tempFile.getName(); + + lowestDescendant = TEST_ROOT_DIR + File.separator + "lowestDescendantPidFile"; + + // write to shell-script + try { + FileWriter fWriter = new FileWriter(shellScript); + fWriter.write( + "# rogue task\n" + + "sleep 1\n" + + "echo hello\n" + + "if [ $1 -ne 0 ]\n" + + "then\n" + + " sh " + shellScript + " $(($1-1))\n" + + "else\n" + + " echo $$ > " + lowestDescendant + "\n" + + " while true\n do\n" + + " sleep 5\n" + + " done\n" + + "fi"); + fWriter.close(); + } catch (IOException ioe) { + LOG.info("Error: " + ioe); + return; + } + + Thread t = new RogueTaskThread(); + t.start(); + String pid = getRogueTaskPID(); + LOG.info("Root process pid: " + pid); + SmapsBasedProcessTree p = createProcessTree(pid); + p.updateProcessTree(); // initialize + LOG.info("ProcessTree: " + p.toString()); + + File leaf = new File(lowestDescendant); + //wait till lowest descendant process of Rougue Task starts execution + while (!leaf.exists()) { + try { + Thread.sleep(500); + } catch (InterruptedException ie) { + break; + } + } + + p.updateProcessTree(); // reconstruct + LOG.info("ProcessTree: " + p.toString()); + + // Get the process-tree dump + String processTreeDump = p.getProcessTreeDump(); + + // destroy the process and all its subprocesses + destroyProcessTree(pid); + + boolean isAlive = true; + for (int tries = 100; tries > 0; tries--) { + if (isSetsidAvailable()) {// whole processtree + isAlive = isAnyProcessInTreeAlive(p); + } else {// process + isAlive = isAlive(pid); + } + if (!isAlive) { + break; + } + Thread.sleep(100); + } + if (isAlive) { + fail("ProcessTree shouldn't be alive"); + } + + LOG.info("Process-tree dump follows: \n" + processTreeDump); + Assert.assertTrue("Process-tree dump doesn't start with a proper header", + processTreeDump.startsWith("\t|- PID PPID PGRPID SESSID CMD_NAME " + + "USER_MODE_TIME(MILLIS) SYSTEM_TIME(MILLIS) VMEM_USAGE(BYTES) " + + "RSSMEM_USAGE(PAGES) FULL_CMD_LINE\n")); + for (int i = N; i >= 0; i--) { + String cmdLineDump = "\\|- [0-9]+ [0-9]+ [0-9]+ [0-9]+ \\(sh\\)" + + " [0-9]+ [0-9]+ [0-9]+ [0-9]+ sh " + shellScript + " " + i; + Pattern pat = Pattern.compile(cmdLineDump); + Matcher mat = pat.matcher(processTreeDump); + Assert.assertTrue("Process-tree dump doesn't contain the cmdLineDump of " + i + + "th process!", mat.find()); + } + + // Not able to join thread sometimes when forking with large N. + try { + t.join(2000); + LOG.info("RogueTaskThread successfully joined."); + } catch (InterruptedException ie) { + LOG.info("Interrupted while joining RogueTaskThread."); + } + + // ProcessTree is gone now. Any further calls should be sane. + p.updateProcessTree(); + Assert.assertFalse("ProcessTree must have been gone", isAlive(pid)); + Assert.assertTrue("Cumulative vmem for the gone-process is " + + p.getCumulativeVmem() + " . It should be zero.", p + .getCumulativeVmem() == 0); + Assert.assertTrue(p.toString().equals("[ ]")); + } + + protected SmapsBasedProcessTree createProcessTree(String pid) { + return new SmapsBasedProcessTree(pid); + } + + protected SmapsBasedProcessTree createProcessTree(String pid, String procfsRootDir) { + return new SmapsBasedProcessTree(pid, procfsRootDir); + } + + protected void destroyProcessTree(String pid) throws IOException { + sendSignal(pid, 9); + } + + /** + * Get PID from a pid-file. + * + * @param pidFileName + * Name of the pid-file. + * @return the PID string read from the pid-file. Returns null if the + * pidFileName points to a non-existing file or if read fails from the + * file. + */ + public static String getPidFromPidFile(String pidFileName) { + BufferedReader pidFile = null; + FileReader fReader = null; + String pid = null; + + try { + fReader = new FileReader(pidFileName); + pidFile = new BufferedReader(fReader); + } catch (FileNotFoundException f) { + LOG.debug("PidFile doesn't exist : " + pidFileName); + return pid; + } + + try { + pid = pidFile.readLine(); + } catch (IOException i) { + LOG.error("Failed to read from " + pidFileName); + } finally { + try { + if (fReader != null) { + fReader.close(); + } + try { + if (pidFile != null) { + pidFile.close(); + } + } catch (IOException i) { + LOG.warn("Error closing the stream " + pidFile); + } + } catch (IOException i) { + LOG.warn("Error closing the stream " + fReader); + } + } + return pid; + } + + class ProcessMemInfo { + private String pid; + private List moduleMemList; + + public ProcessMemInfo(String pid, MemoryMappingInfo[] memInfo) { + this.pid = pid; + this.moduleMemList = new LinkedList(); + for(MemoryMappingInfo info : memInfo) { + moduleMemList.add(info); + } + } + + public List getModuleMemList() { + return moduleMemList; + } + + public String getPid() { + return pid; + } + + public String toString() { + StringBuilder sb = new StringBuilder(); + for (MemoryMappingInfo info : moduleMemList) { + sb.append(LINE_SEPARATOR); + sb.append(info.toString()); + } + return sb.toString(); + } + } + + + class MemoryMappingInfo { + String address; + String size; + String rss; + String pss; + String shared_clean; + String shared_dirty; + String private_clean; + String private_dirty; + String referenced; + String anonymous; + String anonHugePages; + String swap; + String kernelPageSize; + String mmuPageSize; + + public MemoryMappingInfo(String address, String[] entries) { + this.address = address; + size = entries[0]; + rss = entries[1]; + pss = entries[2]; + shared_clean = entries[3]; + shared_dirty = entries[4]; + private_clean = entries[5]; + private_dirty = entries[6]; + referenced = entries[7]; + anonymous = entries[8]; + anonHugePages = entries[9]; + swap = entries[10]; + kernelPageSize = entries[11]; + mmuPageSize = entries[12]; + } + + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(address).append(LINE_SEPARATOR); + sb.append("Size: ").append(size).append(" kB\n"); + sb.append("Rss: ").append(rss).append(" kB\n"); + sb.append("Pss: ").append(pss).append(" kB\n"); + sb.append("Shared_Clean: ").append(shared_clean).append(" kB\n"); + sb.append("Shared_Dirty: ").append(shared_dirty).append(" kB\n"); + sb.append("Private_Clean: ").append(private_clean).append(" kB\n"); + sb.append("Private_Dirty: ").append(private_dirty).append(" kB\n"); + sb.append("Referenced: ").append(referenced).append(" kB\n"); + sb.append("Anonymous: ").append(anonymous).append(" kB\n"); + sb.append("AnonHugePages: ").append(anonHugePages).append(" kB\n"); + sb.append("Swap: ").append(swap).append(" kB\n"); + sb.append("KernelPageSize: ").append(kernelPageSize).append(" kB\n"); + sb.append("MMUPageSize: ").append(mmuPageSize).append(" kB\n"); + return sb.toString(); + } + + } + + public static class ProcessStatInfo { + // sample stat in a single line : 3910 (gpm) S 1 3910 3910 0 -1 4194624 + // 83 0 0 0 0 0 0 0 16 0 1 0 7852 2408448 88 4294967295 134512640 + // 134590050 3220521392 3220520036 10975138 0 0 4096 134234626 + // 4294967295 0 0 17 1 0 0 + String pid; + String name; + String ppid; + String pgrpId; + String session; + String vmem = "0"; + String rssmemPage = "0"; + String utime = "0"; + String stime = "0"; + + public ProcessStatInfo(String[] statEntries) { + pid = statEntries[0]; + name = statEntries[1]; + ppid = statEntries[2]; + pgrpId = statEntries[3]; + session = statEntries[4]; + vmem = statEntries[5]; + if (statEntries.length > 6) { + rssmemPage = statEntries[6]; + } + if (statEntries.length > 7) { + utime = statEntries[7]; + stime = statEntries[8]; + } + } + + // construct a line that mimics the procfs stat file. + // all unused numerical entries are set to 0. + public String getStatLine() { + return String.format("%s (%s) S %s %s %s 0 0 0" + + " 0 0 0 0 %s %s 0 0 0 0 0 0 0 %s %s 0 0" + + " 0 0 0 0 0 0 0 0" + + " 0 0 0 0 0", + pid, name, ppid, pgrpId, session, + utime, stime, vmem, rssmemPage); + } + } + + /** + * A basic test that creates a few process directories and writes + * stat files. Verifies that the cpu time and memory is correctly + * computed. + * @throws IOException if there was a problem setting up the + * fake procfs directories or files. + */ + @Test (timeout = 30000) + public void testCpuAndMemoryForProcessTree() throws IOException { + + // test processes + String[] pids = { "100", "200", "300", "400" }; + // create the fake procfs root directory. + File procfsRootDir = new File(TEST_ROOT_DIR, "proc"); + + try { + setupProcfsRootDir(procfsRootDir); + setupPidDirs(procfsRootDir, pids); + + // create stat objects. + // assuming processes 100, 200, 300 are in tree and 400 is not. + ProcessStatInfo[] procInfos = new ProcessStatInfo[4]; + procInfos[0] = new ProcessStatInfo(new String[] + {"100", "proc1", "1", "100", "100", "100000", "100", "1000", "200"}); + procInfos[1] = new ProcessStatInfo(new String[] + {"200", "proc2", "100", "100", "100", "200000", "200", "2000", "400"}); + procInfos[2] = new ProcessStatInfo(new String[] + {"300", "proc3", "200", "100", "100", "300000", "300", "3000", "600"}); + procInfos[3] = new ProcessStatInfo(new String[] + {"400", "proc4", "1", "400", "400", "400000", "400", "4000", "800"}); + + //Create + MemoryMappingInfo[] moduleMemInfos = new MemoryMappingInfo[4]; + moduleMemInfos[0] = new MemoryMappingInfo("7f56c177c000-7f56c177d000 " + + "rw-p 00010000 08:02 40371558 " + + "/grid/0/jdk1.7.0_25/jre/lib/amd64/libnio.so", new String[] + {"4", "4", "25", "4", "25", "15", "10", "4", "0", "0", "0", "4", "4"}); + moduleMemInfos[1] = new MemoryMappingInfo("7fb09382e000-7fb09382f000 r--s 00003000 " + + "08:02 25953545", new String[] + {"4", "4", "25", "4", "0", "15", "10", "4", "0", "0", "0", "4", "4"}); + moduleMemInfos[2] = new MemoryMappingInfo("7e8790000-7e8b80000 r-xs 00000000 00:00 0", new String[] + {"4", "4", "25", "4", "0", "15", "10", "4", "0", "0", "0", "4", "4"}); + moduleMemInfos[3] = new MemoryMappingInfo("7da677000-7e0dcf000 rw-p 00000000 00:00 0", new String[] + {"4", "4", "25", "4", "50", "15", "10", "4", "0", "0", "0", "4", "4"}); + + ProcessMemInfo[] memInfo = new ProcessMemInfo[4]; + memInfo[0] = new ProcessMemInfo("100", moduleMemInfos); + memInfo[1] = new ProcessMemInfo("200", moduleMemInfos); + memInfo[2] = new ProcessMemInfo("300", moduleMemInfos); + memInfo[3] = new ProcessMemInfo("400", moduleMemInfos); + + writeStatFiles(procfsRootDir, pids, procInfos, memInfo); + + + // crank up the process tree class. + SmapsBasedProcessTree processTree = + createProcessTree("100", procfsRootDir.getAbsolutePath()); + // build the process tree. + processTree.updateProcessTree(); + + // verify cumulative memory + Assert.assertEquals("Cumulative virtual memory does not match", 600000L, + processTree.getCumulativeVmem()); + + // RSS=Min(shared_dirty,PSS)+PrivateClean+PrivateDirty (exclude r-xs, r--s) + Assert.assertEquals("Cumulative rss memory does not match", + (100*1024*3), processTree.getCumulativeRssmem()); + + // verify cumulative cpu time + long cumuCpuTime = ProcfsBasedProcessTree.JIFFY_LENGTH_IN_MILLIS > 0 ? + 7200L * ProcfsBasedProcessTree.JIFFY_LENGTH_IN_MILLIS : 0L; + Assert.assertEquals("Cumulative cpu time does not match", + cumuCpuTime, processTree.getCumulativeCpuTime()); + + // test the cpu time again to see if it cumulates + procInfos[0] = new ProcessStatInfo(new String[] + {"100", "proc1", "1", "100", "100", "100000", "100", "2000", "300"}); + procInfos[1] = new ProcessStatInfo(new String[] + {"200", "proc2", "100", "100", "100", "200000", "200", "3000", "500"}); + + memInfo[0] = new ProcessMemInfo("100", moduleMemInfos); + memInfo[1] = new ProcessMemInfo("100", moduleMemInfos); + + + writeStatFiles(procfsRootDir, pids, procInfos, memInfo); + + // build the process tree. + processTree.updateProcessTree(); + + // verify cumulative cpu time again + cumuCpuTime = ProcfsBasedProcessTree.JIFFY_LENGTH_IN_MILLIS > 0 ? + 9400L * ProcfsBasedProcessTree.JIFFY_LENGTH_IN_MILLIS : 0L; + Assert.assertEquals("Cumulative cpu time does not match", + cumuCpuTime, processTree.getCumulativeCpuTime()); + } finally { + FileUtil.fullyDelete(procfsRootDir); + } + } + + /** + * Tests that cumulative memory is computed only for + * processes older than a given age. + * @throws IOException if there was a problem setting up the + * fake procfs directories or files. + */ + @Test (timeout = 30000) + public void testMemForOlderProcesses() throws IOException { + // initial list of processes + String[] pids = { "100", "200", "300", "400" }; + // create the fake procfs root directory. + File procfsRootDir = new File(TEST_ROOT_DIR, "proc"); + + try { + setupProcfsRootDir(procfsRootDir); + setupPidDirs(procfsRootDir, pids); + + // create stat objects. + // assuming 100, 200 and 400 are in tree, 300 is not. + ProcessStatInfo[] procInfos = new ProcessStatInfo[4]; + procInfos[0] = new ProcessStatInfo(new String[] + {"100", "proc1", "1", "100", "100", "100000", "100"}); + procInfos[1] = new ProcessStatInfo(new String[] + {"200", "proc2", "100", "100", "100", "200000", "200"}); + procInfos[2] = new ProcessStatInfo(new String[] + {"300", "proc3", "1", "300", "300", "300000", "300"}); + procInfos[3] = new ProcessStatInfo(new String[] + {"400", "proc4", "100", "100", "100", "400000", "400"}); + + //Create memory mapping info. 2 of them has r--s, r-xs permission + //which needs to be ignored during RSS calculation + MemoryMappingInfo[] moduleMemInfos = new MemoryMappingInfo[4]; + moduleMemInfos[0] = new MemoryMappingInfo("7f56c177c000-7f56c177d000 " + + "rw-p 00010000 08:02 40371558 " + + "/grid/0/jdk1.7.0_25/jre/lib/amd64/libnio.so", new String[] + {"4", "4", "25", "4", "25", "15", "10", "4", "0", "0", "0", "4", "4"}); + moduleMemInfos[1] = new MemoryMappingInfo("7fb09382e000-7fb09382f000 r--s 00003000 " + + "08:02 25953545", new String[] + {"4", "4", "25", "4", "0", "15", "10", "4", "0", "0", "0", "4", "4"}); + moduleMemInfos[2] = new MemoryMappingInfo("7e8790000-7e8b80000 r-xs 00000000 00:00 0", new String[] + {"4", "4", "25", "4", "0", "15", "10", "4", "0", "0", "0", "4", "4"}); + moduleMemInfos[3] = new MemoryMappingInfo("7da677000-7e0dcf000 rw-p 00000000 00:00 0", new String[] + {"4", "4", "25", "4", "50", "15", "10", "4", "0", "0", "0", "4", "4"}); + + ProcessMemInfo[] memInfo = new ProcessMemInfo[4]; + memInfo[0] = new ProcessMemInfo("100", moduleMemInfos); + memInfo[1] = new ProcessMemInfo("200", moduleMemInfos); + memInfo[2] = new ProcessMemInfo("300", moduleMemInfos); + memInfo[3] = new ProcessMemInfo("400", moduleMemInfos); + + writeStatFiles(procfsRootDir, pids, procInfos, memInfo); + + // crank up the process tree class. + SmapsBasedProcessTree processTree = + createProcessTree("100", procfsRootDir.getAbsolutePath()); + + // build the process tree. + processTree.updateProcessTree(); + + // verify cumulative memory + Assert.assertEquals("Vmem cumulative memory does not match", + 700000L, processTree.getCumulativeVmem()); + //Verify RSS memory + //min(shared_dirty, pss) + private_dirty + private_clean + Assert.assertEquals("RSS cumulative memory does not match", + 100*1024*3, processTree.getCumulativeRssmem()); + + + // write one more process as child of 100. + String[] newPids = { "500" }; + setupPidDirs(procfsRootDir, newPids); + + ProcessStatInfo[] newProcInfos = new ProcessStatInfo[1]; + newProcInfos[0] = new ProcessStatInfo(new String[] + {"500", "proc5", "100", "100", "100", "500000", "500"}); + + memInfo= new ProcessMemInfo[1]; + memInfo[0] = new ProcessMemInfo("500", moduleMemInfos); + + writeStatFiles(procfsRootDir, newPids, newProcInfos, memInfo); + + // check memory includes the new process. + processTree.updateProcessTree(); + + + Assert.assertEquals("Cumulative vmem does not include new process", + 1200000L, processTree.getCumulativeVmem()); + Assert.assertEquals("Cumulative rssmem does not include new process", + 100*1024*4, processTree.getCumulativeRssmem()); + + // however processes older than 1 iteration will retain the older value + Assert.assertEquals("Cumulative vmem shouldn't have included new process", + 700000L, processTree.getCumulativeVmem(1)); + Assert.assertEquals("Cumulative rssmem shouldn't have included new process", + 100*1024*3, processTree.getCumulativeRssmem(1)); + + // one more process + newPids = new String[]{ "600" }; + setupPidDirs(procfsRootDir, newPids); + + newProcInfos = new ProcessStatInfo[1]; + newProcInfos[0] = new ProcessStatInfo(new String[] + {"600", "proc6", "100", "100", "100", "600000", "600"}); + memInfo = new ProcessMemInfo[1]; + memInfo[0] = new ProcessMemInfo("600", moduleMemInfos); + writeStatFiles(procfsRootDir, newPids, newProcInfos, memInfo); + + // refresh process tree + processTree.updateProcessTree(); + + // processes older than 2 iterations should be same as before. + Assert.assertEquals("Cumulative vmem shouldn't have included new processes", + 700000L, processTree.getCumulativeVmem(2)); + Assert.assertEquals("Cumulative rssmem shouldn't have included new processes", + 100*1024*3, processTree.getCumulativeRssmem(2)); + + // processes older than 1 iteration should not include new process, + // but include process 500 + Assert.assertEquals("Cumulative vmem shouldn't have included new processes", + 1200000L, processTree.getCumulativeVmem(1)); + Assert.assertEquals("Cumulative rssmem shouldn't have included new processes", + 100*1024*4, processTree.getCumulativeRssmem(1)); + + // no processes older than 3 iterations, this should be 0 + Assert.assertEquals("Getting non-zero vmem for processes older than 3 iterations", + 0L, processTree.getCumulativeVmem(3)); + Assert.assertEquals("Getting non-zero rssmem for processes older than 3 iterations", + 0L, processTree.getCumulativeRssmem(3)); + } finally { + FileUtil.fullyDelete(procfsRootDir); + } + } + + /** + * Verifies ProcfsBasedProcessTree.checkPidPgrpidForMatch() in case of + * 'constructProcessInfo() returning null' by not writing stat file for the + * mock process + * @throws IOException if there was a problem setting up the + * fake procfs directories or files. + */ + @Test (timeout = 30000) + public void testDestroyProcessTree() throws IOException { + // test process + String pid = "100"; + // create the fake procfs root directory. + File procfsRootDir = new File(TEST_ROOT_DIR, "proc"); + + try { + setupProcfsRootDir(procfsRootDir); + + // crank up the process tree class. + createProcessTree(pid, procfsRootDir.getAbsolutePath()); + + // Let us not create stat file for pid 100. + Assert.assertTrue(ProcfsBasedProcessTree.checkPidPgrpidForMatch( + pid, procfsRootDir.getAbsolutePath())); + } finally { + FileUtil.fullyDelete(procfsRootDir); + } + } + + /** + * Test the correctness of process-tree dump. + * + * @throws IOException + */ + @Test (timeout = 30000) + public void testProcessTreeDump() + throws IOException { + + String[] pids = { "100", "200", "300", "400", "500", "600" }; + + File procfsRootDir = new File(TEST_ROOT_DIR, "proc"); + + try { + setupProcfsRootDir(procfsRootDir); + setupPidDirs(procfsRootDir, pids); + + int numProcesses = pids.length; + // Processes 200, 300, 400 and 500 are descendants of 100. 600 is not. + ProcessStatInfo[] procInfos = new ProcessStatInfo[numProcesses]; + procInfos[0] = new ProcessStatInfo(new String[] { + "100", "proc1", "1", "100", "100", "100000", "100", "1000", "200"}); + procInfos[1] = new ProcessStatInfo(new String[] { + "200", "proc2", "100", "100", "100", "200000", "200", "2000", "400"}); + procInfos[2] = new ProcessStatInfo(new String[] { + "300", "proc3", "200", "100", "100", "300000", "300", "3000", "600"}); + procInfos[3] = new ProcessStatInfo(new String[] { + "400", "proc4", "200", "100", "100", "400000", "400", "4000", "800"}); + procInfos[4] = new ProcessStatInfo(new String[] { + "500", "proc5", "400", "100", "100", "400000", "400", "4000", "800"}); + procInfos[5] = new ProcessStatInfo(new String[] { + "600", "proc6", "1", "1", "1", "400000", "400", "4000", "800"}); + + + //Create memory mapping info. One of them has r--s permission + //which needs to be ignored during RSS calculation + MemoryMappingInfo[] moduleMemInfos = new MemoryMappingInfo[4]; + moduleMemInfos[0] = new MemoryMappingInfo("7f56c177c000-7f56c177d000 " + + "rw-p 00010000 08:02 40371558 " + + "/grid/0/jdk1.7.0_25/jre/lib/amd64/libnio.so", new String[] + {"4", "4", "25", "4", "25", "15", "10", "4", "0", "0", "0", "4", "4"}); + moduleMemInfos[1] = new MemoryMappingInfo("7fb09382e000-7fb09382f000 r--s 00003000 " + + "08:02 25953545", new String[] + {"4", "4", "25", "4", "0", "15", "10", "4", "0", "0", "0", "4", "4"}); + moduleMemInfos[2] = new MemoryMappingInfo("7e8790000-7e8b80000 r-xs 00000000 00:00 0", new String[] + {"4", "4", "25", "4", "0", "15", "10", "4", "0", "0", "0", "4", "4"}); + moduleMemInfos[3] = new MemoryMappingInfo("7da677000-7e0dcf000 rw-p 00000000 00:00 0", new String[] + {"4", "4", "25", "4", "50", "15", "10", "4", "0", "0", "0", "4", "4"}); + + ProcessMemInfo[] memInfo = new ProcessMemInfo[6]; + memInfo[0] = new ProcessMemInfo("100", moduleMemInfos); + memInfo[1] = new ProcessMemInfo("200", moduleMemInfos); + memInfo[2] = new ProcessMemInfo("300", moduleMemInfos); + memInfo[3] = new ProcessMemInfo("400", moduleMemInfos); + memInfo[4] = new ProcessMemInfo("500", moduleMemInfos); + memInfo[5] = new ProcessMemInfo("600", moduleMemInfos); + + + String[] cmdLines = new String[numProcesses]; + cmdLines[0] = "proc1 arg1 arg2"; + cmdLines[1] = "proc2 arg3 arg4"; + cmdLines[2] = "proc3 arg5 arg6"; + cmdLines[3] = "proc4 arg7 arg8"; + cmdLines[4] = "proc5 arg9 arg10"; + cmdLines[5] = "proc6 arg11 arg12"; + + writeStatFiles(procfsRootDir, pids, procInfos, memInfo); + writeCmdLineFiles(procfsRootDir, pids, cmdLines); + + SmapsBasedProcessTree processTree = createProcessTree( + "100", procfsRootDir.getAbsolutePath()); + // build the process tree. + processTree.updateProcessTree(); + + // Get the process-tree dump + String processTreeDump = processTree.getProcessTreeDump(); + + LOG.info("Process-tree dump follows: \n" + processTreeDump); + Assert.assertTrue("Process-tree dump doesn't start with a proper header", + processTreeDump.startsWith("\t|- PID PPID PGRPID SESSID CMD_NAME " + + "USER_MODE_TIME(MILLIS) SYSTEM_TIME(MILLIS) VMEM_USAGE(BYTES) " + + "RSSMEM_USAGE(PAGES) FULL_CMD_LINE\n")); + for (int i = 0; i < 5; i++) { + ProcessStatInfo p = procInfos[i]; + Assert.assertTrue( + "Process-tree dump doesn't contain the cmdLineDump of process " + + p.pid, processTreeDump.contains("\t|- " + p.pid + " " + + p.ppid + " " + p.pgrpId + " " + p.session + " (" + p.name + + ") " + p.utime + " " + p.stime + " " + p.vmem + " " + + p.rssmemPage + " " + cmdLines[i])); + } + + // 600 should not be in the dump + ProcessStatInfo p = procInfos[5]; + Assert.assertFalse( + "Process-tree dump shouldn't contain the cmdLineDump of process " + + p.pid, processTreeDump.contains("\t|- " + p.pid + " " + p.ppid + + " " + p.pgrpId + " " + p.session + " (" + p.name + ") " + + p.utime + " " + p.stime + " " + p.vmem + " " + cmdLines[5])); + } finally { + FileUtil.fullyDelete(procfsRootDir); + } + } + + protected static boolean isSetsidAvailable() { + ShellCommandExecutor shexec = null; + boolean setsidSupported = true; + try { + String[] args = {"setsid", "bash", "-c", "echo $$"}; + shexec = new ShellCommandExecutor(args); + shexec.execute(); + } catch (IOException ioe) { + LOG.warn("setsid is not available on this machine. So not using it."); + setsidSupported = false; + } finally { // handle the exit code + LOG.info("setsid exited with exit code " + shexec.getExitCode()); + } + return setsidSupported; + } + + /** + * Is the root-process alive? + * Used only in tests. + * @return true if the root-process is alive, false otherwise. + */ + private static boolean isAlive(String pid) { + try { + final String sigpid = isSetsidAvailable() ? "-" + pid : pid; + try { + sendSignal(sigpid, 0); + } catch (ExitCodeException e) { + return false; + } + return true; + } catch (IOException ignored) { + } + return false; + } + + private static void sendSignal(String pid, int signal) throws IOException { + ShellCommandExecutor shexec = null; + String[] arg = { "kill", "-" + signal, pid }; + shexec = new ShellCommandExecutor(arg); + shexec.execute(); + } + + /** + * Is any of the subprocesses in the process-tree alive? + * Used only in tests. + * @return true if any of the processes in the process-tree is + * alive, false otherwise. + */ + private static boolean isAnyProcessInTreeAlive( + SmapsBasedProcessTree processTree) { + for (String pId : processTree.getCurrentProcessIDs()) { + if (isAlive(pId)) { + return true; + } + } + return false; + } + + /** + * Create a directory to mimic the procfs file system's root. + * @param procfsRootDir root directory to create. + * @throws IOException if could not delete the procfs root directory + */ + public static void setupProcfsRootDir(File procfsRootDir) + throws IOException { + // cleanup any existing process root dir. + if (procfsRootDir.exists()) { + Assert.assertTrue(FileUtil.fullyDelete(procfsRootDir)); + } + + // create afresh + Assert.assertTrue(procfsRootDir.mkdirs()); + } + + /** + * Create PID directories under the specified procfs root directory + * @param procfsRootDir root directory of procfs file system + * @param pids the PID directories to create. + * @throws IOException If PID dirs could not be created + */ + public static void setupPidDirs(File procfsRootDir, String[] pids) + throws IOException { + for (String pid : pids) { + File pidDir = new File(procfsRootDir, pid); + pidDir.mkdir(); + if (!pidDir.exists()) { + throw new IOException ("couldn't make process directory under " + + "fake procfs"); + } else { + LOG.info("created pid dir"); + } + } + } + + /** + * Write stat files under the specified pid directories with data + * setup in the corresponding ProcessStatInfo objects + * @param procfsRootDir root directory of procfs file system + * @param pids the PID directories under which to create the stat file + * @param procs corresponding ProcessStatInfo objects whose data should be + * written to the stat files. + * @param smap related information + * @throws IOException if stat files could not be written + */ + public static void writeStatFiles(File procfsRootDir, String[] pids, + ProcessStatInfo[] procs, + ProcessMemInfo[] smaps) throws IOException { + for (int i=0; i