diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/CgroupsLCEResourcesHandler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/CgroupsLCEResourcesHandler.java index 63039d8..af0acfa 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/CgroupsLCEResourcesHandler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/CgroupsLCEResourcesHandler.java @@ -57,11 +57,13 @@ private String cgroupMountPath; private boolean cpuWeightEnabled = true; + private boolean vdisksWeightEnabled = true; private boolean strictResourceUsageMode = false; private final String MTAB_FILE = "/proc/mounts"; private final String CGROUPS_FSTYPE = "cgroup"; private final String CONTROLLER_CPU = "cpu"; + private final String CONTROLLER_VDISKS = "blkio"; private final String CPU_PERIOD_US = "cfs_period_us"; private final String CPU_QUOTA_US = "cfs_quota_us"; private final int CPU_DEFAULT_WEIGHT = 1024; // set by kernel @@ -135,6 +137,8 @@ void init(LinuxContainerExecutor lce, ResourceCalculatorPlugin plugin) ArrayList cgroupKVs = new ArrayList(); cgroupKVs.add(CONTROLLER_CPU + "=" + cgroupMountPath + "/" + CONTROLLER_CPU); + cgroupKVs.add(CONTROLLER_VDISKS + "=" + cgroupMountPath + "/" + + CONTROLLER_VDISKS); lce.mountCgroups(cgroupKVs, cgroupPrefix); } @@ -211,6 +215,10 @@ boolean isCpuWeightEnabled() { return this.cpuWeightEnabled; } + boolean isVdisksWeightEnabled() { + return this.vdisksWeightEnabled; + } + /* * Next four functions are for an individual cgroup. */ @@ -318,12 +326,25 @@ private void setupLimits(ContainerId containerId, } } } + if (isVdisksWeightEnabled()) { + createCgroup(CONTROLLER_VDISKS, containerName); + // The allowed weight is [100, 1000] for cgroups + int nodeVdisks = conf.getInt(YarnConfiguration.NM_DISK_VDISKS, + YarnConfiguration.DEFAULT_NM_DISK_VDISKS); + int vdisksShares = 100 + + (int)((900.0) * containerResource.getVirtualDisks() / nodeVdisks); + updateCgroup(CONTROLLER_VDISKS, containerName, "weight", + String.valueOf(vdisksShares)); + } } private void clearLimits(ContainerId containerId) { if (isCpuWeightEnabled()) { deleteCgroup(pathForCgroup(CONTROLLER_CPU, containerId.toString())); } + if (isVdisksWeightEnabled()) { + deleteCgroup(pathForCgroup(CONTROLLER_VDISKS, containerId.toString())); + } } /* @@ -349,6 +370,11 @@ public String getResourcesOption(ContainerId containerId) { sb.append(","); } + if (isVdisksWeightEnabled()) { + sb.append(pathForCgroup(CONTROLLER_VDISKS, containerName) + "/tasks"); + sb.append(","); + } + if (sb.charAt(sb.length() - 1) == ',') { sb.deleteCharAt(sb.length() - 1); } @@ -358,6 +384,7 @@ public String getResourcesOption(ContainerId containerId) { /* We are looking for entries of the form: * none /cgroup/path/mem cgroup rw,memory 0 0 + * none /cgroup/path/blkio cgroup rw,relatime,blkio 0 0 * * Use a simple pattern that splits on the five spaces, and * grabs the 2, 3, and 4th fields. @@ -422,9 +449,7 @@ private void initializeControllerPaths() throws IOException { Map> parsedMtab = parseMtab(); // CPU - controllerPath = findControllerInMtab(CONTROLLER_CPU, parsedMtab); - if (controllerPath != null) { File f = new File(controllerPath + "/" + this.cgroupPrefix); @@ -438,6 +463,22 @@ private void initializeControllerPaths() throws IOException { throw new IOException("Not able to enforce cpu weights; cannot find " + "cgroup for cpu controller in " + getMtabFileName()); } + + // VDisks + controllerPath = findControllerInMtab(CONTROLLER_VDISKS, parsedMtab); + if (controllerPath != null) { + File f = new File(controllerPath + "/" + this.cgroupPrefix); + + if (FileUtil.canWrite(f)) { + controllerPaths.put(CONTROLLER_VDISKS, controllerPath); + } else { + throw new IOException("Not able to enforce vdisks weights; cannot write " + + "to cgroup at: " + controllerPath); + } + } else { + throw new IOException("Not able to enforce vdisks weights; cannot find " + + "cgroup for blkio controller in " + MTAB_FILE); + } } @VisibleForTesting diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestCgroupsLCEResourcesHandler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestCgroupsLCEResourcesHandler.java index d0bceee..d3b1d83 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestCgroupsLCEResourcesHandler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestCgroupsLCEResourcesHandler.java @@ -123,7 +123,8 @@ public void testInit() throws IOException { // create mock cgroup File cgroupDir = createMockCgroup(); - File cgroupMountDir = createMockCgroupMount(cgroupDir); + File cpuCgroupMountDir = createMockCgroupMount(cgroupDir, "cpu"); + File blkioCgroupMountDir = createMockCgroupMount(cgroupDir, "blkio"); // create mock mtab File mockMtab = createMockMTab(cgroupDir); @@ -135,8 +136,8 @@ public void testInit() throws IOException { // in this case, we're using all cpu so the files // shouldn't exist(because init won't create them handler.init(mockLCE, plugin); - File periodFile = new File(cgroupMountDir, "cpu.cfs_period_us"); - File quotaFile = new File(cgroupMountDir, "cpu.cfs_quota_us"); + File periodFile = new File(cpuCgroupMountDir, "cpu.cfs_period_us"); + File quotaFile = new File(cpuCgroupMountDir, "cpu.cfs_quota_us"); Assert.assertFalse(periodFile.exists()); Assert.assertFalse(quotaFile.exists()); @@ -211,9 +212,11 @@ private File createMockCgroup() throws IOException { return cgroupDir; } - private File createMockCgroupMount(File cgroupDir) throws IOException { - File cgroupMountDir = new File(cgroupDir.getAbsolutePath(), "hadoop-yarn"); - if (!cgroupMountDir.mkdir()) { + private File createMockCgroupMount(File cgroupDir, String type) + throws IOException { + File cgroupMountDir = new File(cgroupDir.getAbsolutePath(), + type + "/hadoop-yarn"); + if (!cgroupMountDir.mkdirs()) { String message = "Could not create dir " + cgroupMountDir.getAbsolutePath(); throw new IOException(message); @@ -222,8 +225,11 @@ private File createMockCgroupMount(File cgroupDir) throws IOException { } private File createMockMTab(File cgroupDir) throws IOException { - String mtabContent = - "none " + cgroupDir.getAbsolutePath() + " cgroup rw,relatime,cpu 0 0"; + String cpuMtabContent = + "none " + cgroupDir.getAbsolutePath() + "/cpu cgroup rw,relatime,cpu 0 0\n"; + String blkioMtabContent = + "none " + cgroupDir.getAbsolutePath() + "/blkio cgroup rw,relatime,blkio 0 0\n"; + File mockMtab = new File("target", UUID.randomUUID().toString()); if (!mockMtab.exists()) { if (!mockMtab.createNewFile()) { @@ -232,7 +238,8 @@ private File createMockMTab(File cgroupDir) throws IOException { } } FileWriter mtabWriter = new FileWriter(mockMtab.getAbsoluteFile()); - mtabWriter.write(mtabContent); + mtabWriter.write(cpuMtabContent); + mtabWriter.write(blkioMtabContent); mtabWriter.close(); mockMtab.deleteOnExit(); return mockMtab; @@ -245,6 +252,7 @@ public void testContainerLimits() throws IOException { new CustomCgroupsLCEResourceHandler(); handler.generateLimitsMode = true; YarnConfiguration conf = new YarnConfiguration(); + conf.setInt(YarnConfiguration.NM_DISK_VDISKS, 8); final int numProcessors = 4; ResourceCalculatorPlugin plugin = Mockito.mock(ResourceCalculatorPlugin.class); @@ -254,7 +262,8 @@ public void testContainerLimits() throws IOException { // create mock cgroup File cgroupDir = createMockCgroup(); - File cgroupMountDir = createMockCgroupMount(cgroupDir); + File cpuCgroupMountDir = createMockCgroupMount(cgroupDir, "cpu"); + File blkioCgroupMountDir = createMockCgroupMount(cgroupDir, "blkio"); // create mock mtab File mockMtab = createMockMTab(cgroupDir); @@ -263,62 +272,75 @@ public void testContainerLimits() throws IOException { handler.setMtabFile(mockMtab.getAbsolutePath()); handler.init(mockLCE, plugin); + // check the blkio weight settings + ContainerId id = ContainerId.fromString("container_1_1_1_1"); + handler.preExecute(id, Resource.newInstance(1024, 1, 1)); + File containerBlkioDir = new File(blkioCgroupMountDir, id.toString()); + Assert.assertTrue(containerBlkioDir.exists()); + Assert.assertTrue(containerBlkioDir.isDirectory()); + File blkioWeightFile = new File(containerBlkioDir, "blkio.weight"); + Assert.assertTrue(blkioWeightFile.exists()); + // The weight is calculated as (100 + 900.0 * containerVdisk / nodeVdisks) + Assert.assertEquals(212, readIntFromFile(blkioWeightFile)); + FileUtils.deleteQuietly(containerBlkioDir); + // check values // default case - files shouldn't exist, strict mode off by default - ContainerId id = ContainerId.fromString("container_1_1_1_1"); - handler.preExecute(id, Resource.newInstance(1024, 1)); - File containerDir = new File(cgroupMountDir, id.toString()); - Assert.assertTrue(containerDir.exists()); - Assert.assertTrue(containerDir.isDirectory()); - File periodFile = new File(containerDir, "cpu.cfs_period_us"); - File quotaFile = new File(containerDir, "cpu.cfs_quota_us"); + File containerCpuDir = new File(cpuCgroupMountDir, id.toString()); + Assert.assertTrue(containerCpuDir.exists()); + Assert.assertTrue(containerCpuDir.isDirectory()); + File periodFile = new File(containerCpuDir, "cpu.cfs_period_us"); + File quotaFile = new File(containerCpuDir, "cpu.cfs_quota_us"); Assert.assertFalse(periodFile.exists()); Assert.assertFalse(quotaFile.exists()); // no files created because we're using all cpu - FileUtils.deleteQuietly(containerDir); + FileUtils.deleteQuietly(containerCpuDir); + FileUtils.deleteQuietly(containerBlkioDir); conf.setBoolean( - YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_STRICT_RESOURCE_USAGE, true); + YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_STRICT_RESOURCE_USAGE, true); handler.initConfig(); handler.preExecute(id, - Resource.newInstance(1024, YarnConfiguration.DEFAULT_NM_VCORES)); - Assert.assertTrue(containerDir.exists()); - Assert.assertTrue(containerDir.isDirectory()); - periodFile = new File(containerDir, "cpu.cfs_period_us"); - quotaFile = new File(containerDir, "cpu.cfs_quota_us"); + Resource.newInstance(1024, YarnConfiguration.DEFAULT_NM_VCORES)); + Assert.assertTrue(containerCpuDir.exists()); + Assert.assertTrue(containerCpuDir.isDirectory()); + periodFile = new File(containerCpuDir, "cpu.cfs_period_us"); + quotaFile = new File(containerCpuDir, "cpu.cfs_quota_us"); Assert.assertFalse(periodFile.exists()); Assert.assertFalse(quotaFile.exists()); // 50% of CPU - FileUtils.deleteQuietly(containerDir); + FileUtils.deleteQuietly(containerCpuDir); + FileUtils.deleteQuietly(containerBlkioDir); conf.setBoolean( - YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_STRICT_RESOURCE_USAGE, true); + YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_STRICT_RESOURCE_USAGE, true); handler.initConfig(); handler.preExecute(id, - Resource.newInstance(1024, YarnConfiguration.DEFAULT_NM_VCORES / 2)); - Assert.assertTrue(containerDir.exists()); - Assert.assertTrue(containerDir.isDirectory()); - periodFile = new File(containerDir, "cpu.cfs_period_us"); - quotaFile = new File(containerDir, "cpu.cfs_quota_us"); + Resource.newInstance(1024, YarnConfiguration.DEFAULT_NM_VCORES / 2)); + Assert.assertTrue(containerCpuDir.exists()); + Assert.assertTrue(containerCpuDir.isDirectory()); + periodFile = new File(containerCpuDir, "cpu.cfs_period_us"); + quotaFile = new File(containerCpuDir, "cpu.cfs_quota_us"); Assert.assertTrue(periodFile.exists()); Assert.assertTrue(quotaFile.exists()); Assert.assertEquals(500 * 1000, readIntFromFile(periodFile)); Assert.assertEquals(1000 * 1000, readIntFromFile(quotaFile)); // CGroups set to 50% of CPU, container set to 50% of YARN CPU - FileUtils.deleteQuietly(containerDir); + FileUtils.deleteQuietly(containerCpuDir); + FileUtils.deleteQuietly(containerBlkioDir); conf.setBoolean( - YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_STRICT_RESOURCE_USAGE, true); + YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_STRICT_RESOURCE_USAGE, true); conf .setInt(YarnConfiguration.NM_RESOURCE_PERCENTAGE_PHYSICAL_CPU_LIMIT, 50); handler.initConfig(); handler.init(mockLCE, plugin); handler.preExecute(id, - Resource.newInstance(1024, YarnConfiguration.DEFAULT_NM_VCORES / 2)); - Assert.assertTrue(containerDir.exists()); - Assert.assertTrue(containerDir.isDirectory()); - periodFile = new File(containerDir, "cpu.cfs_period_us"); - quotaFile = new File(containerDir, "cpu.cfs_quota_us"); + Resource.newInstance(1024, YarnConfiguration.DEFAULT_NM_VCORES / 2)); + Assert.assertTrue(containerCpuDir.exists()); + Assert.assertTrue(containerCpuDir.isDirectory()); + periodFile = new File(containerCpuDir, "cpu.cfs_period_us"); + quotaFile = new File(containerCpuDir, "cpu.cfs_quota_us"); Assert.assertTrue(periodFile.exists()); Assert.assertTrue(quotaFile.exists()); Assert.assertEquals(1000 * 1000, readIntFromFile(periodFile));