diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 87d2f0c..6eb6991 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -2172,6 +2172,16 @@ public static boolean isAclEnabled(Configuration conf) { public static final boolean DEFAULT_NM_LINUX_CONTAINER_CGROUPS_STRICT_RESOURCE_USAGE = false; + /** + * Scaling ratio between Cpu time and vCore time when strict mode enabled. + * If -1 scaling is done automatically based on available resource for + * YARN (resource.percentage-physical-cpu-limit). If 1 a vcore will be + * allowed to use 100% of a single physical core time - no scaling + */ + public static final String NM_LINUX_CONTAINER_CGROUPS_STRICT_VCORE_WEIGHT = + NM_PREFIX + "linux-container-executor.cgroups.strict-vcore-weight"; + public static final float DEFAULT_NM_LINUX_CONTAINER_CGROUPS_STRICT_VCORE_WEIGHT = + -1; // Configurations for applicaiton life time monitor feature public static final String RM_APPLICATION_MONITOR_INTERVAL_MS = diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 0835d02..4a512d3 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -1745,7 +1745,15 @@ yarn.nodemanager.linux-container-executor.cgroups.strict-resource-usage false - + + This parameter allows to tune how much cpu time is used by a single + vcore when strict resource mode is enabled by with + yarn.nodemanager.linux-container-executor.cgroups.strict-resource-usage. + By default (-1) automaic vcore scaling is enabled. If set 1 vCore can get up to + one physical/logical core. + yarn.nodemanager.linux-container-executor.cgroups.strict-vcore-weight + -1 + Comma separated list of runtimes that are allowed when using LinuxContainerExecutor. The allowed values are default, docker, and diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsCpuResourceHandlerImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsCpuResourceHandlerImpl.java index e0711df..6521954 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsCpuResourceHandlerImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsCpuResourceHandlerImpl.java @@ -64,6 +64,7 @@ private CGroupsHandler cGroupsHandler; private boolean strictResourceUsageMode = false; + private float vCoreWeight = -1; private float yarnProcessors; private int nodeVCores; private static final CGroupsHandler.CGroupController CPU = @@ -95,6 +96,11 @@ this.strictResourceUsageMode = conf.getBoolean( YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_STRICT_RESOURCE_USAGE, YarnConfiguration.DEFAULT_NM_LINUX_CONTAINER_CGROUPS_STRICT_RESOURCE_USAGE); + this.vCoreWeight = conf.getFloat( + YarnConfiguration + .NM_LINUX_CONTAINER_CGROUPS_STRICT_VCORE_WEIGHT, + YarnConfiguration + .DEFAULT_NM_LINUX_CONTAINER_CGROUPS_STRICT_VCORE_WEIGHT); this.cGroupsHandler.initializeCGroupController(CPU); nodeVCores = NodeManagerHardwareUtils.getVCores(plugin, conf); @@ -227,8 +233,17 @@ public static boolean cpuLimitsExist(String path) } if (strictResourceUsageMode) { if (nodeVCores != containerVCores) { - float containerCPU = - (containerVCores * yarnProcessors) / (float) nodeVCores; + + // Assign CPU time to a container based on requested vCores + float containerCPU = 0; + if (vCoreWeight <= 0) { + // Automatically downscale vCore proportionally to available resources for YARN (default) + containerCPU = (containerVCores * yarnProcessors) / (float) nodeVCores; + } + else { + // Scale container CPU time according to configured vCore weight + containerCPU = Math.min(containerVCores * vCoreWeight, yarnProcessors); + } int[] limits = getOverallLimits(containerCPU); cGroupsHandler.updateCGroupParam(CPU, cgroupId, CGroupsHandler.CGROUP_CPU_PERIOD_US, String.valueOf(limits[0])); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/CgroupsLCEResourcesHandler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/CgroupsLCEResourcesHandler.java index 8d1e933..991b0d2 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/CgroupsLCEResourcesHandler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/CgroupsLCEResourcesHandler.java @@ -79,6 +79,7 @@ private boolean cpuWeightEnabled = true; private boolean strictResourceUsageMode = false; + private float vCoreWeight = -1; private final String MTAB_FILE = "/proc/mounts"; private final String CGROUPS_FSTYPE = "cgroup"; @@ -136,6 +137,14 @@ void initConfig() throws IOException { YarnConfiguration .DEFAULT_NM_LINUX_CONTAINER_CGROUPS_STRICT_RESOURCE_USAGE); + this.vCoreWeight = + conf + .getFloat( + YarnConfiguration + .NM_LINUX_CONTAINER_CGROUPS_STRICT_VCORE_WEIGHT, + YarnConfiguration + .DEFAULT_NM_LINUX_CONTAINER_CGROUPS_STRICT_VCORE_WEIGHT); + int len = cgroupPrefix.length(); if (cgroupPrefix.charAt(len - 1) == '/') { cgroupPrefix = cgroupPrefix.substring(0, len - 1); @@ -337,8 +346,18 @@ private void setupLimits(ContainerId containerId, String.valueOf(cpuShares)); if (strictResourceUsageMode) { if (nodeVCores != containerVCores) { - float containerCPU = - (containerVCores * yarnProcessors) / (float) nodeVCores; + + // Assign CPU time to a container based on requested vCores + float containerCPU = 0; + if (vCoreWeight <= 0) { + // Automatically downscale vCore proportionally to available resources for YARN (default) + containerCPU = (containerVCores * yarnProcessors) / (float) nodeVCores; + } + else { + // Scale container CPU time according to configured vCore weight + containerCPU = Math.min(containerVCores * vCoreWeight, yarnProcessors ); + } + int[] limits = getOverallLimits(containerCPU); updateCgroup(CONTROLLER_CPU, containerName, CPU_PERIOD_US, String.valueOf(limits[0])); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsCpuResourceHandlerImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsCpuResourceHandlerImpl.java index 842fc6b..3ba87a8 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsCpuResourceHandlerImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsCpuResourceHandlerImpl.java @@ -272,6 +272,73 @@ public void testPreStartRestrictedContainers() throws Exception { } @Test + public void testPreStartRestrictedContainersWithWeights() throws Exception { + String id = "container_01_01"; + String path = "test-path/" + id; + int defaultVCores = 8; + float vCoreWeight = .5f; + Configuration conf = new YarnConfiguration(); + conf.setBoolean( + YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_STRICT_RESOURCE_USAGE, + true); + int cpuPerc = 50; + conf.setInt(YarnConfiguration.NM_RESOURCE_PERCENTAGE_PHYSICAL_CPU_LIMIT, + cpuPerc); + conf.setFloat( + YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_STRICT_VCORE_WEIGHT, + vCoreWeight); + + cGroupsCpuResourceHandler.bootstrap(plugin, conf); + verify(mockCGroupsHandler, times(1)) + .updateCGroupParam(CGroupsHandler.CGroupController.CPU, "", + CGroupsHandler.CGROUP_CPU_PERIOD_US, String.valueOf("500000")); + verify(mockCGroupsHandler, times(1)) + .updateCGroupParam(CGroupsHandler.CGroupController.CPU, "", + CGroupsHandler.CGROUP_CPU_QUOTA_US, + String.valueOf(CGroupsCpuResourceHandlerImpl.MAX_QUOTA_US)); + float yarnCores = (cpuPerc * numProcessors) / 100; + int[] containerVCores = { 1, 4 }; + for (int cVcores : containerVCores) { + ContainerId mockContainerId = mock(ContainerId.class); + when(mockContainerId.toString()).thenReturn(id); + Container mockContainer = mock(Container.class); + when(mockContainer.getContainerId()).thenReturn(mockContainerId); + when(mockCGroupsHandler + .getPathForCGroupTasks(CGroupsHandler.CGroupController.CPU, id)) + .thenReturn(path); + when(mockContainer.getResource()) + .thenReturn(Resource.newInstance(1024, cVcores)); + when(mockCGroupsHandler + .getPathForCGroupTasks(CGroupsHandler.CGroupController.CPU, id)) + .thenReturn(path); + + float share = Math.min(cVcores * vCoreWeight, yarnCores); + int quotaUS; + int periodUS; + if (share >= 1.0f) { + quotaUS = CGroupsCpuResourceHandlerImpl.MAX_QUOTA_US; + periodUS = + (int) ((float) CGroupsCpuResourceHandlerImpl.MAX_QUOTA_US / share ); + } else { + quotaUS = (int) (CGroupsCpuResourceHandlerImpl.MAX_QUOTA_US * share); + periodUS = CGroupsCpuResourceHandlerImpl.MAX_QUOTA_US; + } + cGroupsCpuResourceHandler.preStart(mockContainer); + verify(mockCGroupsHandler, times(1)) + .updateCGroupParam(CGroupsHandler.CGroupController.CPU, id, + CGroupsHandler.CGROUP_CPU_SHARES, String.valueOf( + CGroupsCpuResourceHandlerImpl.CPU_DEFAULT_WEIGHT * cVcores)); + // set quota and period + verify(mockCGroupsHandler, times(1)) + .updateCGroupParam(CGroupsHandler.CGroupController.CPU, id, + CGroupsHandler.CGROUP_CPU_PERIOD_US, String.valueOf(periodUS)); + verify(mockCGroupsHandler, times(1)) + .updateCGroupParam(CGroupsHandler.CGroupController.CPU, id, + CGroupsHandler.CGROUP_CPU_QUOTA_US, String.valueOf(quotaUS)); + } + } + + @Test public void testReacquireContainer() throws Exception { ContainerId containerIdMock = mock(ContainerId.class); Assert.assertNull( diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestCgroupsLCEResourcesHandler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestCgroupsLCEResourcesHandler.java index 7d8704f..5f1b2ca 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestCgroupsLCEResourcesHandler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestCgroupsLCEResourcesHandler.java @@ -391,6 +391,87 @@ public void testSelectCgroup() { } } +@Test + public void testVCoreWeightLimits() throws IOException { + LinuxContainerExecutor mockLCE = new MockLinuxContainerExecutor(); + CustomCgroupsLCEResourceHandler handler = + new CustomCgroupsLCEResourceHandler(); + handler.generateLimitsMode = true; + YarnConfiguration conf = new YarnConfiguration(); + conf.setBoolean(YarnConfiguration.NM_DISK_RESOURCE_ENABLED, true); + final int numProcessors = 4; + ResourceCalculatorPlugin plugin = + Mockito.mock(ResourceCalculatorPlugin.class); + Mockito.doReturn(numProcessors).when(plugin).getNumProcessors(); + Mockito.doReturn(numProcessors).when(plugin).getNumCores(); + handler.setConf(conf); + handler.initConfig(); + + // create mock mtab + File mockMtab = + TestCGroupsHandlerImpl.createPremountedCgroups(cgroupDir, false); + + // create mock cgroup + File cpuCgroupMountDir = createMockCgroupMount( + cgroupDir, "cpu"); + + // setup our handler and call init() + handler.setMtabFile(mockMtab.getAbsolutePath()); + handler.init(mockLCE, plugin); + File periodFile = new File(cpuCgroupMountDir, "cpu.cfs_period_us"); + File quotaFile = new File(cpuCgroupMountDir, "cpu.cfs_quota_us"); + Assert.assertFalse(periodFile.exists()); + Assert.assertFalse(quotaFile.exists()); + + // check the controller paths map isn't empty + ContainerId id = ContainerId.fromString("container_1_1_1_1"); + File containerCpuDir = new File(cpuCgroupMountDir, id.toString()); + + // vCore wieght 0.5 + conf.setBoolean( + YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_STRICT_RESOURCE_USAGE, + true); + conf.setFloat( + YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_STRICT_VCORE_WEIGHT, + .5f); + handler.initConfig(); + handler.preExecute(id, + Resource.newInstance(1024, YarnConfiguration.DEFAULT_NM_VCORES/2)); + Assert.assertTrue(containerCpuDir.exists()); + Assert.assertTrue(containerCpuDir.isDirectory()); + periodFile = new File(containerCpuDir, "cpu.cfs_period_us"); + quotaFile = new File(containerCpuDir, "cpu.cfs_quota_us"); + Assert.assertTrue(periodFile.exists()); + Assert.assertTrue(quotaFile.exists()); + Assert.assertEquals(500 * 1000, readIntFromFile(periodFile)); + Assert.assertEquals(1000 * 1000, readIntFromFile(quotaFile)); + + // No CPU downscaling when reduced resource + // - weight set to 1, CPU-limit 50% + FileUtils.deleteQuietly(containerCpuDir); + conf.setBoolean( + YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_STRICT_RESOURCE_USAGE, + true); + conf.setFloat( + YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_STRICT_VCORE_WEIGHT, + 1); + conf + .setInt(YarnConfiguration.NM_RESOURCE_PERCENTAGE_PHYSICAL_CPU_LIMIT, 50); + handler.initConfig(); + handler.preExecute(id, + Resource.newInstance(1024, YarnConfiguration.DEFAULT_NM_VCORES/2)); + Assert.assertTrue(containerCpuDir.exists()); + Assert.assertTrue(containerCpuDir.isDirectory()); + periodFile = new File(containerCpuDir, "cpu.cfs_period_us"); + quotaFile = new File(containerCpuDir, "cpu.cfs_quota_us"); + Assert.assertTrue(periodFile.exists()); + Assert.assertTrue(quotaFile.exists()); + Assert.assertEquals(250 * 1000, readIntFromFile(periodFile)); + Assert.assertEquals(1000 * 1000, readIntFromFile(quotaFile)); + + FileUtils.deleteQuietly(cgroupDir); + } + @Test public void testManualCgroupSetting() throws IOException { CgroupsLCEResourcesHandler handler = new CgroupsLCEResourcesHandler();