diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java index b2dfb78..fd2c90a 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java @@ -20,6 +20,7 @@ import java.io.BufferedReader; import java.io.File; +import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.FileReader; import java.io.IOException; @@ -32,10 +33,12 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; +import org.apache.commons.io.IOUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.util.Shell; import org.apache.hadoop.util.Shell.ShellCommandExecutor; import org.apache.hadoop.util.StringUtils; @@ -61,7 +64,46 @@ public static final String PROCFS_CMDLINE_FILE = "cmdline"; public static final long PAGE_SIZE; public static final long JIFFY_LENGTH_IN_MILLIS; // in millisecond - + + enum MEM_INFO { + SIZE("Size"), RSS("Rss"), PSS("Pss"), SHARED_CLEAN("Shared_Clean"), SHARED_DIRTY( + "Shared_Dirty"), PRIVATE_CLEAN("Private_Clean"), PRIVATE_DIRTY( + "Private_Dirty"), REFERENCED("Referenced"), ANONYMOUS("Anonymous"), ANON_HUGE_PAGES( + "AnonHugePages"), SWAP("swap"), KERNEL_PAGE_SIZE("kernelPageSize"), MMU_PAGE_SIZE( + "mmuPageSize"), INVALID("invalid"); + + private String name; + + private MEM_INFO(String name) { + this.name = name; + } + + public static MEM_INFO getMemInfoByName(String name) { + for (MEM_INFO info : MEM_INFO.values()) { + if (info.name.trim().equalsIgnoreCase(name.trim())) { + return info; + } + } + return INVALID; + } + } + + public static final String SMAPS = "smaps"; + private static final String KB = "kB"; + private static final String READ_ONLY_WITH_SHARED_PERMISSION = "r--s"; + private static final String READ_EXECUTE_WITH_SHARED_PERMISSION = "r-xs"; + private static final Pattern ADDRESS_PATTERN = Pattern + .compile("([[a-f]|(0-9)]*)-([[a-f]|(0-9)]*)(\\s)*([rxwps\\-]*)"); + private static final Pattern MEM_INFO_PATTERN = Pattern + .compile("(^[A-Z].*):[\\s ]*(.*)"); + + public static final String PROCFS_SMAPS_ENABLED = + "yarn.nodemanager.container-monitor.process-tree.smaps.enabled"; + public static final boolean DEFAULT_PROCFS_SMAPS_ENABLED = false; + private boolean smapsEnabled; + + protected Map processSMAPTree = new HashMap(); + static { long jiffiesPerSecond = -1; long pageSize = -1; @@ -210,6 +252,17 @@ public void updateProcessTree() { // Log.debug the ProcfsBasedProcessTree LOG.debug(this.toString()); } + if (isSmapEnabled()) { + //Update smaps info + for (ProcessInfo p : processTree.values()) { + if (p != null) { + // Get information for each process + ProcessMemInfo memInfo = new ProcessMemInfo(p.getPid()); + constructProcessSMAPInfo(memInfo, procfsDir); + processSMAPTree.put(p.getPid(), memInfo); + } + } + } } } @@ -297,6 +350,9 @@ public long getCumulativeVmem(int olderThanAge) { */ @Override public long getCumulativeRssmem(int olderThanAge) { + if (isSmapEnabled()) { + return getSmapBasedCumulativeRssmem(olderThanAge); + } if (PAGE_SIZE < 0) { return 0; } @@ -309,6 +365,62 @@ public long getCumulativeRssmem(int olderThanAge) { return totalPages * PAGE_SIZE; // convert # pages to byte } + public boolean isSmapEnabled() { + Configuration conf = getConf(); + if (conf != null) { + return conf.getBoolean(PROCFS_SMAPS_ENABLED, DEFAULT_PROCFS_SMAPS_ENABLED); + } else { + return false; + } + } + + /** + * Get the cumulative resident set size (RSS) memory used by all the processes + * in the process-tree that are older than the passed in age. RSS is + * calculated based on SMAP information. Skip mappings with "r--s", "r-xs" + * permissions to get real RSS usage of the process. + * + * + * @param olderThanAge + * processes above this age are included in the memory addition + * @return cumulative rss memory used by the process-tree in bytes, for + * processes older than this age. return 0 if it cannot be calculated + */ + public long getSmapBasedCumulativeRssmem(int olderThanAge) { + long total = 0; + for (ProcessInfo p : processTree.values()) { + if ((p != null) && (p.getAge() > olderThanAge)) { + ProcessMemInfo procMemInfo = processSMAPTree.get(p.getPid()); + if (procMemInfo != null) { + for (MemoryMappingInfo info : procMemInfo.getModuleMemList()) { + // Do not account for r--s or r-xs mappings + if (info.getPermission().trim() + .equalsIgnoreCase(READ_ONLY_WITH_SHARED_PERMISSION) || + info.getPermission().trim() + .equalsIgnoreCase(READ_EXECUTE_WITH_SHARED_PERMISSION)) { + continue; + } + total += Math.min(info.sharedDirty, info.pss) + info.privateDirty + + info.privateClean; + if (LOG.isDebugEnabled()) { + LOG.debug(" total(" + olderThanAge + "): PID : " + p.getPid() + + ", SharedDirty : " + info.sharedDirty + ", PSS : " + + info.pss + ", Private_Dirty : " + info.privateDirty + + ", Private_Clean : " + info.privateClean + ", total : " + + (total * 1024)); + } + } + } + if (LOG.isDebugEnabled()) { + LOG.debug(procMemInfo.toString()); + } + } + } + total = (total * 1024); // convert to bytes + LOG.info("SmapBasedCumulativeRssmem (bytes) : " + total); + return total; // size + } + /** * Get the CPU time in millisecond used by all the processes in the * process-tree since the process-tree created @@ -601,4 +713,242 @@ public String getCmdLine(String procfsDir) { return ret; } } + + /** + * Update memory related information + * + * @param pinfo + * @param procfsDir + */ + private static void constructProcessSMAPInfo(ProcessMemInfo pInfo, + String procfsDir) { + BufferedReader in = null; + FileReader fReader = null; + try { + File pidDir = new File(procfsDir, pInfo.getPid()); + File file = new File(pidDir, SMAPS); + if (!file.exists()) { + return; + } + fReader = new FileReader(file); + in = new BufferedReader(fReader); + MemoryMappingInfo memoryMappingInfo = null; + List lines = IOUtils.readLines(new FileInputStream(file)); + for (String line : lines) { + line = line.trim(); + try { + Matcher address = ADDRESS_PATTERN.matcher(line); + if (address.find()) { + memoryMappingInfo = new MemoryMappingInfo(line); + memoryMappingInfo.setPermission(address.group(4)); + pInfo.getModuleMemList().add(memoryMappingInfo); + continue; + } + Matcher memInfo = MEM_INFO_PATTERN.matcher(line); + if (memInfo.find()) { + String key = memInfo.group(1).trim(); + String value = memInfo.group(2).replace(KB, "").trim(); + if (LOG.isDebugEnabled()) { + LOG.debug("MemInfo : " + key + " : Value : " + value); + } + memoryMappingInfo.updateModuleMemInfo(key, value); + } + } catch (Throwable t) { + LOG.warn("Error parsing smaps line : " + line + "; " + t.getMessage()); + } + } + } catch (FileNotFoundException f) { + LOG.error(f.getMessage()); + } catch (IOException e) { + LOG.error(e.getMessage()); + } catch (Throwable t) { + LOG.error(t.getMessage()); + } finally { + IOUtils.closeQuietly(in); + } + } + + /** + * Placeholder for process's SMAPS information + */ + static class ProcessMemInfo { + private String pid; + private List moduleMemList; + + public ProcessMemInfo(String pid) { + this.pid = pid; + this.moduleMemList = new LinkedList(); + } + + public List getModuleMemList() { + return moduleMemList; + } + + public String getPid() { + return pid; + } + + public String toString() { + StringBuilder sb = new StringBuilder(); + for (MemoryMappingInfo info : moduleMemList) { + sb.append("\n"); + sb.append(info.toString()); + } + return sb.toString(); + } + } + + /** + *
+   * Private Pages : Pages that were mapped only by the process
+   * Shared Pages : Pages that were shared with other processes
+   *
+   * Clean Pages : Pages that have not been modified since they were mapped
+   * Dirty Pages : Pages that have been modified since they were mapped
+   *
+   * Private RSS = Private Clean Pages + Private Dirty Pages
+   * Shared RSS = Shared Clean Pages + Shared Dirty Pages
+   * RSS = Private RSS + Shared RSS
+   * PSS = The count of all pages mapped uniquely by the process, 
+   *  plus a fraction of each shared page, said fraction to be 
+   *  proportional to the number of processes which have mapped the page.
+   *
+   * 
+ */ + static class MemoryMappingInfo { + private int size; + private int rss; + private int pss; + private int sharedClean; + private int sharedDirty; + private int privateClean; + private int privateDirty; + private int referenced; + private String regionName; + private String permission; + + public MemoryMappingInfo(String name) { + this.regionName = name; + } + + public String getName() { + return regionName; + } + + public void setPermission(String permission) { + this.permission = permission; + } + + public String getPermission() { + return permission; + } + + public int getSize() { + return size; + } + + public int getRss() { + return rss; + } + + public int getPss() { + return pss; + } + + public int getSharedClean() { + return sharedClean; + } + + public int getSharedDirty() { + return sharedDirty; + } + + public int getPrivateClean() { + return privateClean; + } + + public int getPrivateDirty() { + return privateDirty; + } + + public int getReferenced() { + return referenced; + } + + public void updateModuleMemInfo(String key, String value) { + MEM_INFO info = MEM_INFO.getMemInfoByName(key); + int val = 0; + try { + val = Integer.parseInt(value.trim()); + } catch (NumberFormatException ne) { + LOG.error("Error in parsing : " + info + " : value" + value.trim()); + } + if (info == null) { + return; + } + if (LOG.isDebugEnabled()) { + LOG.debug("updateModuleMemInfo : memInfo : " + info); + } + switch (info) { + case SIZE: + size = val; + break; + case RSS: + rss = val; + break; + case PSS: + pss = val; + break; + case SHARED_CLEAN: + sharedClean = val; + break; + case SHARED_DIRTY: + sharedDirty = val; + break; + case PRIVATE_CLEAN: + privateClean = val; + break; + case PRIVATE_DIRTY: + privateDirty = val; + break; + case REFERENCED: + referenced = val; + break; + default: + break; + } + } + + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("\t").append(this.getName()).append("\n"); + sb.append("\t").append(MEM_INFO.SIZE.name + ":" + this.getSize()) + .append(" kB\n"); + sb.append("\t").append(MEM_INFO.PSS.name + ":" + this.getPss()) + .append(" kB\n"); + sb.append("\t").append(MEM_INFO.RSS.name + ":" + this.getRss()) + .append(" kB\n"); + sb.append("\t") + .append(MEM_INFO.SHARED_CLEAN.name + ":" + this.getSharedClean()) + .append(" kB\n"); + sb.append("\t") + .append(MEM_INFO.SHARED_DIRTY.name + ":" + this.getSharedDirty()) + .append(" kB\n"); + sb.append("\t") + .append(MEM_INFO.PRIVATE_CLEAN.name + ":" + this.getPrivateClean()) + .append(" kB\n"); + sb.append("\t") + .append(MEM_INFO.PRIVATE_DIRTY.name + ":" + this.getPrivateDirty()) + .append(" kB\n"); + sb.append("\t").append(MEM_INFO.REFERENCED.name + ":" + this.getReferenced()) + .append(" kB\n"); + sb.append("\t") + .append(MEM_INFO.PRIVATE_DIRTY.name + ":" + this.getPrivateDirty()) + .append(" kB\n"); + sb.append("\t") + .append(MEM_INFO.PRIVATE_DIRTY.name + ":" + this.getPrivateDirty()) + .append(" kB\n"); + return sb.toString(); + } + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index deb3c2e..2143268 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -1206,4 +1206,12 @@ yarn.client.application-client-protocol.poll-interval-ms 200 + + + Whether ProcfsBasedProcessTree should use /proc/pid/smaps + for computing RSS usage of a process. + + yarn.nodemanager.container-monitor.process-tree.smaps.enabled + false + diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestProcfsBasedProcessTree.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestProcfsBasedProcessTree.java index 349beeb..77bba36 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestProcfsBasedProcessTree.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestProcfsBasedProcessTree.java @@ -27,6 +27,7 @@ import java.io.FileReader; import java.io.FileWriter; import java.io.IOException; +import java.util.List; import java.util.Random; import java.util.Vector; import java.util.regex.Matcher; @@ -34,14 +35,17 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileContext; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.util.Shell; import org.apache.hadoop.util.Shell.ExitCodeException; import org.apache.hadoop.util.Shell.ShellCommandExecutor; -import org.apache.hadoop.yarn.util.ProcfsBasedProcessTree; +import org.apache.hadoop.util.StringUtils; +import org.apache.hadoop.yarn.util.ProcfsBasedProcessTree.MEM_INFO; +import org.apache.hadoop.yarn.util.ProcfsBasedProcessTree.MemoryMappingInfo; +import org.apache.hadoop.yarn.util.ProcfsBasedProcessTree.ProcessMemInfo; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -337,6 +341,46 @@ public String getStatLine() { } } + public MemoryMappingInfo constructMemoryMappingInfo(String address, String[] entries) { + MemoryMappingInfo info = new MemoryMappingInfo(address); + info.updateModuleMemInfo(MEM_INFO.SIZE.name(), entries[0]); + info.updateModuleMemInfo(MEM_INFO.RSS.name(), entries[1]); + info.updateModuleMemInfo(MEM_INFO.PSS.name(), entries[2]); + info.updateModuleMemInfo(MEM_INFO.SHARED_CLEAN.name(), entries[3]); + info.updateModuleMemInfo(MEM_INFO.SHARED_DIRTY.name(), entries[4]); + info.updateModuleMemInfo(MEM_INFO.PRIVATE_CLEAN.name(), entries[5]); + info.updateModuleMemInfo(MEM_INFO.PRIVATE_DIRTY.name(), entries[6]); + info.updateModuleMemInfo(MEM_INFO.REFERENCED.name(), entries[7]); + info.updateModuleMemInfo(MEM_INFO.ANONYMOUS.name(), entries[8]); + info.updateModuleMemInfo(MEM_INFO.ANON_HUGE_PAGES.name(), entries[9]); + info.updateModuleMemInfo(MEM_INFO.SWAP.name(), entries[10]); + info.updateModuleMemInfo(MEM_INFO.KERNEL_PAGE_SIZE.name(), entries[11]); + info.updateModuleMemInfo(MEM_INFO.MMU_PAGE_SIZE.name(), entries[12]); + return info; + } + + public void createMemoryMappingInfo(ProcessMemInfo[] procMemInfo) { + for(int i=0;i memoryMappingList = procMemInfo[i].getModuleMemList(); + memoryMappingList.add(constructMemoryMappingInfo("7f56c177c000-7f56c177d000 " + + "rw-p 00010000 08:02 40371558 " + + "/grid/0/jdk1.7.0_25/jre/lib/amd64/libnio.so", new String[] + {"4", "4", "25", "4", "25", "15", "10", "4", "0", "0", "0", "4", "4"})); + memoryMappingList.add(constructMemoryMappingInfo("7fb09382e000-7fb09382f000 r--s 00003000 " + + "08:02 25953545", new String[] + {"4", "4", "25", "4", "0", "15", "10", "4", "0", "0", "0", "4", "4"})); + memoryMappingList.add(constructMemoryMappingInfo("7e8790000-7e8b80000 r-xs 00000000 00:00 0", + new String[] + {"4", "4", "25", "4", "0", "15", "10", "4", "0", "0", "0", "4", "4"})); + memoryMappingList.add(constructMemoryMappingInfo("7da677000-7e0dcf000 rw-p 00000000 00:00 0", + new String[] + {"4", "4", "25", "4", "50", "15", "10", "4", "0", "0", "0", "4", "4"})); + } + } + /** * A basic test that creates a few process directories and writes * stat files. Verifies that the cpu time and memory is correctly @@ -368,11 +412,19 @@ public void testCpuAndMemoryForProcessTree() throws IOException { procInfos[3] = new ProcessStatInfo(new String[] {"400", "proc4", "1", "400", "400", "400000", "400", "4000", "800"}); - writeStatFiles(procfsRootDir, pids, procInfos); + ProcessMemInfo[] memInfo = new ProcessMemInfo[4]; + memInfo[0] = new ProcessMemInfo("100"); + memInfo[1] = new ProcessMemInfo("200"); + memInfo[2] = new ProcessMemInfo("300"); + memInfo[3] = new ProcessMemInfo("400"); + createMemoryMappingInfo(memInfo); + writeStatFiles(procfsRootDir, pids, procInfos, memInfo); // crank up the process tree class. + Configuration conf = new Configuration(); ProcfsBasedProcessTree processTree = createProcessTree("100", procfsRootDir.getAbsolutePath()); + processTree.setConf(conf); // build the process tree. processTree.updateProcessTree(); @@ -391,13 +443,18 @@ public void testCpuAndMemoryForProcessTree() throws IOException { 7200L * ProcfsBasedProcessTree.JIFFY_LENGTH_IN_MILLIS : 0L; Assert.assertEquals("Cumulative cpu time does not match", cumuCpuTime, processTree.getCumulativeCpuTime()); + //Check by enabling smaps + setSmapsInProceTree(processTree, true); + // RSS=Min(shared_dirty,PSS)+PrivateClean+PrivateDirty (exclude r-xs, r--s) + Assert.assertEquals("Cumulative rss memory does not match", + (100*1024*3), processTree.getCumulativeRssmem()); // test the cpu time again to see if it cumulates procInfos[0] = new ProcessStatInfo(new String[] {"100", "proc1", "1", "100", "100", "100000", "100", "2000", "300"}); procInfos[1] = new ProcessStatInfo(new String[] {"200", "proc2", "100", "100", "100", "200000", "200", "3000", "500"}); - writeStatFiles(procfsRootDir, pids, procInfos); + writeStatFiles(procfsRootDir, pids, procInfos, memInfo); // build the process tree. processTree.updateProcessTree(); @@ -412,6 +469,13 @@ public void testCpuAndMemoryForProcessTree() throws IOException { } } + private void setSmapsInProceTree(ProcfsBasedProcessTree processTree, + boolean enableFlag) { + processTree.getConf(). + setBoolean(ProcfsBasedProcessTree.PROCFS_SMAPS_ENABLED, enableFlag); + processTree.updateProcessTree(); + } + /** * Tests that cumulative memory is computed only for * processes older than a given age. @@ -420,6 +484,11 @@ public void testCpuAndMemoryForProcessTree() throws IOException { */ @Test (timeout = 30000) public void testMemForOlderProcesses() throws IOException { + testMemForOlderProcesses(false); + testMemForOlderProcesses(true); + } + + private void testMemForOlderProcesses(boolean smapEnabled) throws IOException { // initial list of processes String[] pids = { "100", "200", "300", "400" }; // create the fake procfs root directory. @@ -440,19 +509,25 @@ public void testMemForOlderProcesses() throws IOException { {"300", "proc3", "1", "300", "300", "300000", "300"}); procInfos[3] = new ProcessStatInfo(new String[] {"400", "proc4", "100", "100", "100", "400000", "400"}); - - writeStatFiles(procfsRootDir, pids, procInfos); + //write smap information invariably for testing + ProcessMemInfo[] memInfo = new ProcessMemInfo[4]; + memInfo[0] = new ProcessMemInfo("100"); + memInfo[1] = new ProcessMemInfo("200"); + memInfo[2] = new ProcessMemInfo("300"); + memInfo[3] = new ProcessMemInfo("400"); + createMemoryMappingInfo(memInfo); + writeStatFiles(procfsRootDir, pids, procInfos, memInfo); // crank up the process tree class. + Configuration conf = new Configuration(); ProcfsBasedProcessTree processTree = createProcessTree("100", procfsRootDir.getAbsolutePath()); - // build the process tree. - processTree.updateProcessTree(); + processTree.setConf(conf); + setSmapsInProceTree(processTree, smapEnabled); // verify cumulative memory Assert.assertEquals("Cumulative memory does not match", 700000L, processTree.getCumulativeVmem()); - // write one more process as child of 100. String[] newPids = { "500" }; setupPidDirs(procfsRootDir, newPids); @@ -460,24 +535,37 @@ public void testMemForOlderProcesses() throws IOException { ProcessStatInfo[] newProcInfos = new ProcessStatInfo[1]; newProcInfos[0] = new ProcessStatInfo(new String[] {"500", "proc5", "100", "100", "100", "500000", "500"}); - writeStatFiles(procfsRootDir, newPids, newProcInfos); + ProcessMemInfo[] newMemInfos = new ProcessMemInfo[1]; + newMemInfos[0] = new ProcessMemInfo("500"); + createMemoryMappingInfo(newMemInfos); + writeStatFiles(procfsRootDir, newPids, newProcInfos, newMemInfos); // check memory includes the new process. processTree.updateProcessTree(); Assert.assertEquals("Cumulative vmem does not include new process", 1200000L, processTree.getCumulativeVmem()); - long cumuRssMem = ProcfsBasedProcessTree.PAGE_SIZE > 0 ? + if (!smapEnabled) { + long cumuRssMem = ProcfsBasedProcessTree.PAGE_SIZE > 0 ? 1200L * ProcfsBasedProcessTree.PAGE_SIZE : 0L; - Assert.assertEquals("Cumulative rssmem does not include new process", + Assert.assertEquals("Cumulative rssmem does not include new process", cumuRssMem, processTree.getCumulativeRssmem()); + } else { + Assert.assertEquals("Cumulative rssmem does not include new process", + 100*1024*4, processTree.getCumulativeRssmem()); + } // however processes older than 1 iteration will retain the older value Assert.assertEquals("Cumulative vmem shouldn't have included new process", 700000L, processTree.getCumulativeVmem(1)); - cumuRssMem = ProcfsBasedProcessTree.PAGE_SIZE > 0 ? + if (!smapEnabled) { + long cumuRssMem = ProcfsBasedProcessTree.PAGE_SIZE > 0 ? 700L * ProcfsBasedProcessTree.PAGE_SIZE : 0L; - Assert.assertEquals("Cumulative rssmem shouldn't have included new process", + Assert.assertEquals("Cumulative rssmem shouldn't have included new process", cumuRssMem, processTree.getCumulativeRssmem(1)); + } else { + Assert.assertEquals("Cumulative rssmem shouldn't have included new process", + 100*1024*3, processTree.getCumulativeRssmem(1)); + } // one more process newPids = new String[]{ "600" }; @@ -486,7 +574,10 @@ public void testMemForOlderProcesses() throws IOException { newProcInfos = new ProcessStatInfo[1]; newProcInfos[0] = new ProcessStatInfo(new String[] {"600", "proc6", "100", "100", "100", "600000", "600"}); - writeStatFiles(procfsRootDir, newPids, newProcInfos); + newMemInfos = new ProcessMemInfo[1]; + newMemInfos[0] = new ProcessMemInfo("600"); + createMemoryMappingInfo(newMemInfos); + writeStatFiles(procfsRootDir, newPids, newProcInfos, newMemInfos); // refresh process tree processTree.updateProcessTree(); @@ -494,25 +585,37 @@ public void testMemForOlderProcesses() throws IOException { // processes older than 2 iterations should be same as before. Assert.assertEquals("Cumulative vmem shouldn't have included new processes", 700000L, processTree.getCumulativeVmem(2)); - cumuRssMem = ProcfsBasedProcessTree.PAGE_SIZE > 0 ? + if (!smapEnabled) { + long cumuRssMem = ProcfsBasedProcessTree.PAGE_SIZE > 0 ? 700L * ProcfsBasedProcessTree.PAGE_SIZE : 0L; - Assert.assertEquals("Cumulative rssmem shouldn't have included new processes", + Assert.assertEquals("Cumulative rssmem shouldn't have included new processes", cumuRssMem, processTree.getCumulativeRssmem(2)); + } else { + Assert.assertEquals("Cumulative rssmem shouldn't have included new processes", + 100*1024*3, processTree.getCumulativeRssmem(2)); + } // processes older than 1 iteration should not include new process, // but include process 500 Assert.assertEquals("Cumulative vmem shouldn't have included new processes", 1200000L, processTree.getCumulativeVmem(1)); - cumuRssMem = ProcfsBasedProcessTree.PAGE_SIZE > 0 ? + if (!smapEnabled) { + long cumuRssMem = ProcfsBasedProcessTree.PAGE_SIZE > 0 ? 1200L * ProcfsBasedProcessTree.PAGE_SIZE : 0L; - Assert.assertEquals("Cumulative rssmem shouldn't have included new processes", + Assert.assertEquals("Cumulative rssmem shouldn't have included new processes", cumuRssMem, processTree.getCumulativeRssmem(1)); + } else { + Assert.assertEquals("Cumulative rssmem shouldn't have included new processes", + 100*1024*4, processTree.getCumulativeRssmem(1)); + } // no processes older than 3 iterations, this should be 0 Assert.assertEquals("Getting non-zero vmem for processes older than 3 iterations", 0L, processTree.getCumulativeVmem(3)); Assert.assertEquals("Getting non-zero rssmem for processes older than 3 iterations", 0L, processTree.getCumulativeRssmem(3)); + Assert.assertEquals("Getting non-zero rssmem for processes older than 3 iterations", + 0L, processTree.getCumulativeRssmem(3)); } finally { FileUtil.fullyDelete(procfsRootDir); } @@ -579,6 +682,14 @@ public void testProcessTreeDump() procInfos[5] = new ProcessStatInfo(new String[] { "600", "proc6", "1", "1", "1", "400000", "400", "4000", "800"}); + ProcessMemInfo[] memInfos = new ProcessMemInfo[6]; + memInfos[0] = new ProcessMemInfo("100"); + memInfos[1] = new ProcessMemInfo("200"); + memInfos[2] = new ProcessMemInfo("300"); + memInfos[3] = new ProcessMemInfo("400"); + memInfos[4] = new ProcessMemInfo("500"); + memInfos[5] = new ProcessMemInfo("600"); + String[] cmdLines = new String[numProcesses]; cmdLines[0] = "proc1 arg1 arg2"; cmdLines[1] = "proc2 arg3 arg4"; @@ -587,7 +698,8 @@ public void testProcessTreeDump() cmdLines[4] = "proc5 arg9 arg10"; cmdLines[5] = "proc6 arg11 arg12"; - writeStatFiles(procfsRootDir, pids, procInfos); + createMemoryMappingInfo(memInfos); + writeStatFiles(procfsRootDir, pids, procInfos, memInfos); writeCmdLineFiles(procfsRootDir, pids, cmdLines); ProcfsBasedProcessTree processTree = createProcessTree( @@ -729,7 +841,7 @@ public static void setupPidDirs(File procfsRootDir, String[] pids) * @throws IOException if stat files could not be written */ public static void writeStatFiles(File procfsRootDir, String[] pids, - ProcessStatInfo[] procs) throws IOException { + ProcessStatInfo[] procs,ProcessMemInfo[] smaps) throws IOException { for (int i=0; i