diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 86f45b8..bc4e99b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -1611,6 +1611,16 @@ public static boolean isAclEnabled(Configuration conf) { public static final boolean DEFAULT_NM_LINUX_CONTAINER_CGROUPS_STRICT_RESOURCE_USAGE = false; + /** + * Max resource usage percent when apps not run in strict resource usage mode. + * In non-strict resource usage mode, apps can use resource no more than + * maxResourceUsagePercent * containerVCores + */ + public static final String NM_LINUX_CONTAINER_CGROUPS_MAX_RESOURCE_USAGE_PERCENT = + NM_PREFIX + "linux-container-executor.cgroups.max-resource-usage-percent"; + public static final float DEFAULT_NM_LINUX_CONTAINER_CGROUPS_MAX_RESOURCE_USAGE_PERCENT = + 1.0f; + // Configurations for applicaiton life time monitor feature public static final String RM_APPLICATION_MONITOR_INTERVAL_MS = diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index f93de44..972a261 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -1553,6 +1553,13 @@ + This configuration setting determines the maximum of resource when + strict-resource-usage is false. + yarn.nodemanager.linux-container-executor.cgroups.max-resource-usage-percent + 1.0 + + + This configuration setting determines the capabilities assigned to docker containers when they are launched. While these may not be case-sensitive from a docker perspective, it is best to keep these diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsCpuResourceHandlerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsCpuResourceHandlerImpl.java index 830782d..580e957 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsCpuResourceHandlerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsCpuResourceHandlerImpl.java @@ -65,6 +65,8 @@ private boolean strictResourceUsageMode = false; private float yarnProcessors; private int nodeVCores; + private float maxResourceUsagePercentInNonstrictMode = 1.0f; + private float maxResourceUsagePercentInStrictMode = 1.0f; private static final CGroupsHandler.CGroupController CPU = CGroupsHandler.CGroupController.CPU; @@ -94,6 +96,9 @@ this.strictResourceUsageMode = conf.getBoolean( YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_STRICT_RESOURCE_USAGE, YarnConfiguration.DEFAULT_NM_LINUX_CONTAINER_CGROUPS_STRICT_RESOURCE_USAGE); + this.maxResourceUsagePercentInNonstrictMode = conf.getFloat( + YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_MAX_RESOURCE_USAGE_PERCENT, + YarnConfiguration.DEFAULT_NM_LINUX_CONTAINER_CGROUPS_MAX_RESOURCE_USAGE_PERCENT); this.cGroupsHandler.initializeCGroupController(CPU); nodeVCores = NodeManagerHardwareUtils.getVCores(plugin, conf); @@ -201,16 +206,19 @@ public static boolean cpuLimitsExist(String path) .updateCGroupParam(CPU, cgroupId, CGroupsHandler.CGROUP_CPU_SHARES, String.valueOf(cpuShares)); } - if (strictResourceUsageMode) { - if (nodeVCores != containerVCores) { - float containerCPU = - (containerVCores * yarnProcessors) / (float) nodeVCores; - int[] limits = getOverallLimits(containerCPU); - cGroupsHandler.updateCGroupParam(CPU, cgroupId, - CGroupsHandler.CGROUP_CPU_PERIOD_US, String.valueOf(limits[0])); - cGroupsHandler.updateCGroupParam(CPU, cgroupId, - CGroupsHandler.CGROUP_CPU_QUOTA_US, String.valueOf(limits[1])); + if (nodeVCores != containerVCores) { + float containerCPU = + (containerVCores * yarnProcessors) / (float) nodeVCores; + int[] limits; + if (strictResourceUsageMode) { + limits = getOverallLimits(containerCPU * maxResourceUsagePercentInStrictMode); + } else { + limits = getOverallLimits(containerCPU * maxResourceUsagePercentInNonstrictMode); } + cGroupsHandler.updateCGroupParam(CPU, cgroupId, + CGroupsHandler.CGROUP_CPU_PERIOD_US, String.valueOf(limits[0])); + cGroupsHandler.updateCGroupParam(CPU, cgroupId, + CGroupsHandler.CGROUP_CPU_QUOTA_US, String.valueOf(limits[1])); } } catch (ResourceHandlerException re) { cGroupsHandler.deleteCGroup(CPU, cgroupId); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsCpuResourceHandlerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsCpuResourceHandlerImpl.java index 006b060..fbff2bf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsCpuResourceHandlerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsCpuResourceHandlerImpl.java @@ -144,10 +144,10 @@ public void testPreStart() throws Exception { .valueOf(CGroupsCpuResourceHandlerImpl.CPU_DEFAULT_WEIGHT * 2)); // don't set quota or period - verify(mockCGroupsHandler, never()) + verify(mockCGroupsHandler, times(1)) .updateCGroupParam(eq(CGroupsHandler.CGroupController.CPU), eq(id), eq(CGroupsHandler.CGROUP_CPU_PERIOD_US), anyString()); - verify(mockCGroupsHandler, never()) + verify(mockCGroupsHandler, times(1)) .updateCGroupParam(eq(CGroupsHandler.CGroupController.CPU), eq(id), eq(CGroupsHandler.CGROUP_CPU_QUOTA_US), anyString()); Assert.assertNotNull(ret); @@ -162,6 +162,68 @@ public void testPreStart() throws Exception { } @Test + public void testPreStartNonRestrictedContainers() throws Exception { + String id = "container_01_01"; + String path = "test-path/" + id; + int defaultVCores = 8; + float maxResourceUsagePercent = 2.0f; + Configuration conf = new YarnConfiguration(); + conf.setFloat( + YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_MAX_RESOURCE_USAGE_PERCENT, + maxResourceUsagePercent); + int cpuPerc = 75; + conf.setInt(YarnConfiguration.NM_RESOURCE_PERCENTAGE_PHYSICAL_CPU_LIMIT, + cpuPerc); + cGroupsCpuResourceHandler.bootstrap(plugin, conf); + verify(mockCGroupsHandler, times(1)) + .updateCGroupParam(CGroupsHandler.CGroupController.CPU, "", + CGroupsHandler.CGROUP_CPU_PERIOD_US, String.valueOf("333333")); + verify(mockCGroupsHandler, times(1)) + .updateCGroupParam(CGroupsHandler.CGroupController.CPU, "", + CGroupsHandler.CGROUP_CPU_QUOTA_US, + String.valueOf(CGroupsCpuResourceHandlerImpl.MAX_QUOTA_US)); + float yarnCores = (cpuPerc * numProcessors) / 100; + int containerVCores = 2; + + ContainerId mockContainerId = mock(ContainerId.class); + when(mockContainerId.toString()).thenReturn(id); + Container mockContainer = mock(Container.class); + when(mockContainer.getContainerId()).thenReturn(mockContainerId); + when(mockCGroupsHandler + .getPathForCGroupTasks(CGroupsHandler.CGroupController.CPU, id)) + .thenReturn(path); + when(mockContainer.getResource()) + .thenReturn(Resource.newInstance(1024, containerVCores)); + when(mockCGroupsHandler + .getPathForCGroupTasks(CGroupsHandler.CGroupController.CPU, id)) + .thenReturn(path); + + float share = (containerVCores * maxResourceUsagePercent * yarnCores) / defaultVCores; + int quotaUS; + int periodUS; + if (share > 1.0f) { + quotaUS = CGroupsCpuResourceHandlerImpl.MAX_QUOTA_US; + periodUS = + (int) ((float) CGroupsCpuResourceHandlerImpl.MAX_QUOTA_US / share); + } else { + quotaUS = (int) (CGroupsCpuResourceHandlerImpl.MAX_QUOTA_US * share); + periodUS = CGroupsCpuResourceHandlerImpl.MAX_QUOTA_US; + } + cGroupsCpuResourceHandler.preStart(mockContainer); + verify(mockCGroupsHandler, times(1)) + .updateCGroupParam(CGroupsHandler.CGroupController.CPU, id, + CGroupsHandler.CGROUP_CPU_SHARES, String.valueOf( + CGroupsCpuResourceHandlerImpl.CPU_DEFAULT_WEIGHT * containerVCores)); + // set quota and period + verify(mockCGroupsHandler, times(1)) + .updateCGroupParam(CGroupsHandler.CGroupController.CPU, id, + CGroupsHandler.CGROUP_CPU_PERIOD_US, String.valueOf(periodUS)); + verify(mockCGroupsHandler, times(1)) + .updateCGroupParam(CGroupsHandler.CGroupController.CPU, id, + CGroupsHandler.CGROUP_CPU_QUOTA_US, String.valueOf(quotaUS)); + } + + @Test public void testPreStartStrictUsage() throws Exception { String id = "container_01_01"; String path = "test-path/" + id;