diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 87d2f0c..efbf627 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -2172,6 +2172,16 @@ public static boolean isAclEnabled(Configuration conf) {
public static final boolean DEFAULT_NM_LINUX_CONTAINER_CGROUPS_STRICT_RESOURCE_USAGE =
false;
+ /**
+ * Scaling ratio between Cpu time and vCore time when strict mode enabled.
+ * If -1 scaling is done automatically based on available resource for
+ * YARN (resource.percentage-physical-cpu-limit). If 1 a vcore will be
+ * allowed to use 100% of a single physical core time - no scaling
+ */
+ public static final String NM_LINUX_CONTAINER_CGROUPS_STRICT_VCORE_WEIGHT =
+ NM_PREFIX + "linux-container-executor.cgroups.strict-vcore-weight";
+ public static final float DEFAULT_NM_LINUX_CONTAINER_CGROUPS_STRICT_VCORE_WEIGHT =
+ -1;
// Configurations for applicaiton life time monitor feature
public static final String RM_APPLICATION_MONITOR_INTERVAL_MS =
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 0835d02..313588b 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -1745,7 +1745,15 @@
yarn.nodemanager.linux-container-executor.cgroups.strict-resource-usage
false
-
+
+ This parameter allows to tune how much cpu time is used by a single
+ vcore when strict resource mode is enabled by with
+ yarn.nodemanager.linux-container-executor.cgroups.strict-resource-usage.
+ By default (-1) automaic vcore scaling is enabled. If set 1 vCore can get up to
+ one physical/logical core.
+ yarn.nodemanager.linux-container-executor.cgroups.strict-vcore-weight
+ -1
+
Comma separated list of runtimes that are allowed when using
LinuxContainerExecutor. The allowed values are default, docker, and
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsCpuResourceHandlerImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsCpuResourceHandlerImpl.java
index e0711df..cffc829 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsCpuResourceHandlerImpl.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsCpuResourceHandlerImpl.java
@@ -64,6 +64,7 @@
private CGroupsHandler cGroupsHandler;
private boolean strictResourceUsageMode = false;
+ private float vCoreWeight = -1;
private float yarnProcessors;
private int nodeVCores;
private static final CGroupsHandler.CGroupController CPU =
@@ -95,6 +96,11 @@
this.strictResourceUsageMode = conf.getBoolean(
YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_STRICT_RESOURCE_USAGE,
YarnConfiguration.DEFAULT_NM_LINUX_CONTAINER_CGROUPS_STRICT_RESOURCE_USAGE);
+ this.vCoreWeight = conf.getFloat(
+ YarnConfiguration
+ .NM_LINUX_CONTAINER_CGROUPS_STRICT_VCORE_WEIGHT,
+ YarnConfiguration
+ .DEFAULT_NM_LINUX_CONTAINER_CGROUPS_STRICT_VCORE_WEIGHT);
this.cGroupsHandler.initializeCGroupController(CPU);
nodeVCores = NodeManagerHardwareUtils.getVCores(plugin, conf);
@@ -227,8 +233,19 @@ public static boolean cpuLimitsExist(String path)
}
if (strictResourceUsageMode) {
if (nodeVCores != containerVCores) {
- float containerCPU =
- (containerVCores * yarnProcessors) / (float) nodeVCores;
+
+ // Assign CPU time to a container based on requested vCores
+ float containerCPU = 0;
+ if (vCoreWeight <= 0) {
+ // Automatically downscale vCore proportionally
+ // to available resources for YARN (default)
+ containerCPU = (containerVCores * yarnProcessors) /
+ (float) nodeVCores;
+ } else {
+ // Scale container CPU time according to configured vCore weight
+ containerCPU =
+ Math.min(containerVCores * vCoreWeight, yarnProcessors);
+ }
int[] limits = getOverallLimits(containerCPU);
cGroupsHandler.updateCGroupParam(CPU, cgroupId,
CGroupsHandler.CGROUP_CPU_PERIOD_US, String.valueOf(limits[0]));
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/CgroupsLCEResourcesHandler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/CgroupsLCEResourcesHandler.java
index 8d1e933..f6a21d1 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/CgroupsLCEResourcesHandler.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/CgroupsLCEResourcesHandler.java
@@ -79,6 +79,7 @@
private boolean cpuWeightEnabled = true;
private boolean strictResourceUsageMode = false;
+ private float vCoreWeight = -1;
private final String MTAB_FILE = "/proc/mounts";
private final String CGROUPS_FSTYPE = "cgroup";
@@ -136,6 +137,14 @@ void initConfig() throws IOException {
YarnConfiguration
.DEFAULT_NM_LINUX_CONTAINER_CGROUPS_STRICT_RESOURCE_USAGE);
+ this.vCoreWeight =
+ conf
+ .getFloat(
+ YarnConfiguration
+ .NM_LINUX_CONTAINER_CGROUPS_STRICT_VCORE_WEIGHT,
+ YarnConfiguration
+ .DEFAULT_NM_LINUX_CONTAINER_CGROUPS_STRICT_VCORE_WEIGHT);
+
int len = cgroupPrefix.length();
if (cgroupPrefix.charAt(len - 1) == '/') {
cgroupPrefix = cgroupPrefix.substring(0, len - 1);
@@ -337,8 +346,20 @@ private void setupLimits(ContainerId containerId,
String.valueOf(cpuShares));
if (strictResourceUsageMode) {
if (nodeVCores != containerVCores) {
- float containerCPU =
- (containerVCores * yarnProcessors) / (float) nodeVCores;
+
+ // Assign CPU time to a container based on requested vCores
+ float containerCPU = 0;
+ if (vCoreWeight <= 0) {
+ // Automatically downscale vCore proportionally
+ // to available resources for YARN (default)
+ containerCPU = (containerVCores * yarnProcessors) /
+ (float) nodeVCores;
+ } else {
+ // Scale container CPU time according to configured vCore weight
+ containerCPU =
+ Math.min(containerVCores * vCoreWeight, yarnProcessors);
+ }
+
int[] limits = getOverallLimits(containerCPU);
updateCgroup(CONTROLLER_CPU, containerName, CPU_PERIOD_US,
String.valueOf(limits[0]));
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsCpuResourceHandlerImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsCpuResourceHandlerImpl.java
index 842fc6b..a7f4a36 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsCpuResourceHandlerImpl.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsCpuResourceHandlerImpl.java
@@ -272,6 +272,73 @@ public void testPreStartRestrictedContainers() throws Exception {
}
@Test
+ public void testPreStartRestrictedContainersWithWeights() throws Exception {
+ String id = "container_01_01";
+ String path = "test-path/" + id;
+ int defaultVCores = 8;
+ float vCoreWeight = .5f;
+ Configuration conf = new YarnConfiguration();
+ conf.setBoolean(
+ YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_STRICT_RESOURCE_USAGE,
+ true);
+ int cpuPerc = 50;
+ conf.setInt(YarnConfiguration.NM_RESOURCE_PERCENTAGE_PHYSICAL_CPU_LIMIT,
+ cpuPerc);
+ conf.setFloat(
+ YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_STRICT_VCORE_WEIGHT,
+ vCoreWeight);
+
+ cGroupsCpuResourceHandler.bootstrap(plugin, conf);
+ verify(mockCGroupsHandler, times(1))
+ .updateCGroupParam(CGroupsHandler.CGroupController.CPU, "",
+ CGroupsHandler.CGROUP_CPU_PERIOD_US, String.valueOf("500000"));
+ verify(mockCGroupsHandler, times(1))
+ .updateCGroupParam(CGroupsHandler.CGroupController.CPU, "",
+ CGroupsHandler.CGROUP_CPU_QUOTA_US,
+ String.valueOf(CGroupsCpuResourceHandlerImpl.MAX_QUOTA_US));
+ float yarnCores = (cpuPerc * numProcessors) / 100;
+ int[] containerVCores = { 1, 4};
+ for (int cVcores : containerVCores) {
+ ContainerId mockContainerId = mock(ContainerId.class);
+ when(mockContainerId.toString()).thenReturn(id);
+ Container mockContainer = mock(Container.class);
+ when(mockContainer.getContainerId()).thenReturn(mockContainerId);
+ when(mockCGroupsHandler
+ .getPathForCGroupTasks(CGroupsHandler.CGroupController.CPU, id))
+ .thenReturn(path);
+ when(mockContainer.getResource())
+ .thenReturn(Resource.newInstance(1024, cVcores));
+ when(mockCGroupsHandler
+ .getPathForCGroupTasks(CGroupsHandler.CGroupController.CPU, id))
+ .thenReturn(path);
+
+ float share = Math.min(cVcores * vCoreWeight, yarnCores);
+ int quotaUS;
+ int periodUS;
+ if (share >= 1.0f) {
+ quotaUS = CGroupsCpuResourceHandlerImpl.MAX_QUOTA_US;
+ periodUS =
+ (int) ((float) CGroupsCpuResourceHandlerImpl.MAX_QUOTA_US / share);
+ } else {
+ quotaUS = (int) (CGroupsCpuResourceHandlerImpl.MAX_QUOTA_US * share);
+ periodUS = CGroupsCpuResourceHandlerImpl.MAX_QUOTA_US;
+ }
+ cGroupsCpuResourceHandler.preStart(mockContainer);
+ verify(mockCGroupsHandler, times(1))
+ .updateCGroupParam(CGroupsHandler.CGroupController.CPU, id,
+ CGroupsHandler.CGROUP_CPU_SHARES, String.valueOf(
+ CGroupsCpuResourceHandlerImpl.CPU_DEFAULT_WEIGHT * cVcores));
+ // set quota and period
+ verify(mockCGroupsHandler, times(1))
+ .updateCGroupParam(CGroupsHandler.CGroupController.CPU, id,
+ CGroupsHandler.CGROUP_CPU_PERIOD_US, String.valueOf(periodUS));
+ verify(mockCGroupsHandler, times(1))
+ .updateCGroupParam(CGroupsHandler.CGroupController.CPU, id,
+ CGroupsHandler.CGROUP_CPU_QUOTA_US, String.valueOf(quotaUS));
+ }
+ }
+
+ @Test
public void testReacquireContainer() throws Exception {
ContainerId containerIdMock = mock(ContainerId.class);
Assert.assertNull(
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestCgroupsLCEResourcesHandler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestCgroupsLCEResourcesHandler.java
index 7d8704f..aa61967 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestCgroupsLCEResourcesHandler.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestCgroupsLCEResourcesHandler.java
@@ -392,6 +392,87 @@ public void testSelectCgroup() {
}
@Test
+ public void testVCoreWeightLimits() throws IOException {
+ LinuxContainerExecutor mockLCE = new MockLinuxContainerExecutor();
+ CustomCgroupsLCEResourceHandler handler =
+ new CustomCgroupsLCEResourceHandler();
+ handler.generateLimitsMode = true;
+ YarnConfiguration conf = new YarnConfiguration();
+ conf.setBoolean(YarnConfiguration.NM_DISK_RESOURCE_ENABLED, true);
+ final int numProcessors = 4;
+ ResourceCalculatorPlugin plugin =
+ Mockito.mock(ResourceCalculatorPlugin.class);
+ Mockito.doReturn(numProcessors).when(plugin).getNumProcessors();
+ Mockito.doReturn(numProcessors).when(plugin).getNumCores();
+ handler.setConf(conf);
+ handler.initConfig();
+
+ // create mock mtab
+ File mockMtab =
+ TestCGroupsHandlerImpl.createPremountedCgroups(cgroupDir, false);
+
+ // create mock cgroup
+ File cpuCgroupMountDir = createMockCgroupMount(
+ cgroupDir, "cpu");
+
+ // setup our handler and call init()
+ handler.setMtabFile(mockMtab.getAbsolutePath());
+ handler.init(mockLCE, plugin);
+ File periodFile = new File(cpuCgroupMountDir, "cpu.cfs_period_us");
+ File quotaFile = new File(cpuCgroupMountDir, "cpu.cfs_quota_us");
+ Assert.assertFalse(periodFile.exists());
+ Assert.assertFalse(quotaFile.exists());
+
+ // check the controller paths map isn't empty
+ ContainerId id = ContainerId.fromString("container_1_1_1_1");
+ File containerCpuDir = new File(cpuCgroupMountDir, id.toString());
+
+ // vCore wieght 0.5
+ conf.setBoolean(
+ YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_STRICT_RESOURCE_USAGE,
+ true);
+ conf.setFloat(
+ YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_STRICT_VCORE_WEIGHT,
+ .5f);
+ handler.initConfig();
+ handler.preExecute(id,
+ Resource.newInstance(1024, YarnConfiguration.DEFAULT_NM_VCORES/2));
+ Assert.assertTrue(containerCpuDir.exists());
+ Assert.assertTrue(containerCpuDir.isDirectory());
+ periodFile = new File(containerCpuDir, "cpu.cfs_period_us");
+ quotaFile = new File(containerCpuDir, "cpu.cfs_quota_us");
+ Assert.assertTrue(periodFile.exists());
+ Assert.assertTrue(quotaFile.exists());
+ Assert.assertEquals(500 * 1000, readIntFromFile(periodFile));
+ Assert.assertEquals(1000 * 1000, readIntFromFile(quotaFile));
+
+ // No CPU downscaling when reduced resource
+ // - weight set to 1, CPU-limit 50%
+ FileUtils.deleteQuietly(containerCpuDir);
+ conf.setBoolean(
+ YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_STRICT_RESOURCE_USAGE,
+ true);
+ conf.setFloat(
+ YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_STRICT_VCORE_WEIGHT,
+ 1);
+ conf
+ .setInt(YarnConfiguration.NM_RESOURCE_PERCENTAGE_PHYSICAL_CPU_LIMIT, 50);
+ handler.initConfig();
+ handler.preExecute(id,
+ Resource.newInstance(1024, YarnConfiguration.DEFAULT_NM_VCORES/2));
+ Assert.assertTrue(containerCpuDir.exists());
+ Assert.assertTrue(containerCpuDir.isDirectory());
+ periodFile = new File(containerCpuDir, "cpu.cfs_period_us");
+ quotaFile = new File(containerCpuDir, "cpu.cfs_quota_us");
+ Assert.assertTrue(periodFile.exists());
+ Assert.assertTrue(quotaFile.exists());
+ Assert.assertEquals(250 * 1000, readIntFromFile(periodFile));
+ Assert.assertEquals(1000 * 1000, readIntFromFile(quotaFile));
+
+ FileUtils.deleteQuietly(cgroupDir);
+ }
+
+ @Test
public void testManualCgroupSetting() throws IOException {
CgroupsLCEResourcesHandler handler = new CgroupsLCEResourcesHandler();
YarnConfiguration conf = new YarnConfiguration();