diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 03fa1ce..2be9523 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -1611,10 +1611,33 @@ public static boolean isAclEnabled(Configuration conf) { public static final String NM_OVERALLOCATION_MEMORY_UTILIZATION_THRESHOLD = NM_PREFIX + "overallocation.memory-utilization-threshold"; - public static final String NM_OVERALLOCATION_PREEMPTION_THRESHOLD = - NM_PREFIX + "overallocation.preemption-threshold"; - public static final float DEFAULT_NM_OVERALLOCATION_PREEMPTION_THRESHOLD - = 0.96f; + /** + * The CPU utilization threshold, if went beyond for a few times in a row, + * OPPORTUNISTIC containers started due to overallocation should start + * getting preempted. + */ + public static final String NM_OVERALLOCATION_CPU_PREEMPTION_THRESHOLD = + NM_PREFIX + "overallocation.preemption-threshold.cpu"; + public static final float + DEFAULT_NM_OVERALLOCATION_CPU_PREEMPTION_THRESHOLD = 0.99f; + + /** + * The number of times that CPU utilization must go over the CPU preemption + * threshold consecutively before preemption starts to kick in. + */ + public static final String NM_OVERALLOCATION_PREEMPTION_CPU_COUNT = + NM_PREFIX + "overallocation.preemption-threshold-count.cpu"; + public static final int DEFAULT_NM_OVERALLOCATION_PREEMPTION_CPU_COUNT = 4; + + + /** + * The memory utilization threshold beyond which OPPORTUNISTIC containers + * started due to overallocation should start getting preempted. + */ + public static final String NM_OVERALLOCATION_MEMORY_PREEMPTION_THRESHOLD = + NM_PREFIX + "overallocation.preemption-threshold.memory"; + public static final float + DEFAULT_NM_OVERALLOCATION_MEMORY_PREEMPTION_THRESHOLD = 0.95f; /** * Interval of time the linux container executor should try cleaning up diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 42166a9..837e9f3 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -1606,11 +1606,37 @@ When a node is over-allocated to improve utilization by - running OPPORTUNISTIC containers, this config captures the utilization - beyond which OPPORTUNISTIC containers should start getting preempted. + running OPPORTUNISTIC containers, this config captures the CPU + utilization beyond which OPPORTUNISTIC containers should start getting + preempted. This is used in combination with + yarn.nodemanager.overallocation.preemption-threshold-count.cpu, that is, + only when the CPU utilization goes over this threshold consecutively for + a few times will preemption kicks in. - yarn.nodemanager.overallocation.preemption-threshold - 0.96 + yarn.nodemanager.overallocation.preemption-threshold.cpu + 0.99 + + + + When a node is over-allocated to improve utilization by + running OPPORTUNISTIC containers, this config captures the number of + times that CPU utilization has to go above + ${yarn.nodemanager.overallocation.preemption-threshold.cpu} + consecutively for NM to start preempting OPPORTUNISTIC containers + started due to overallocation. + + yarn.nodemanager.overallocation.preemption-threshold-count.cpu + 4 + + + + When a node is over-allocated to improve utilization by + running OPPORTUNISTIC containers, this config captures the CPU + utilization beyond which OPPORTUNISTIC containers should start getting + preempted. + + yarn.nodemanager.overallocation.preemption-threshold.memory + 0.95 diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java index a7cbc5b..1e41803 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java @@ -102,6 +102,7 @@ private ResourceUtilization containersUtilization; private ResourceThresholds overAllocationPreemptionThresholds; + private int overAlloctionPreemptionCpuCount = -1; private volatile boolean stopped = false; @@ -236,7 +237,7 @@ private void initializeOverAllocation(Configuration conf) { YarnConfiguration.MAX_NM_OVERALLOCATION_THRESHOLD); if (overAllocationMemoryUtilizationThreshold <= 0) { LOG.info("NodeManager oversubscription is disabled because the memory " + - "utilization threshold is no larger than zero."); + "overallocation threshold is no larger than zero."); return; } @@ -248,36 +249,49 @@ private void initializeOverAllocation(Configuration conf) { YarnConfiguration.MAX_NM_OVERALLOCATION_THRESHOLD); if (overAllocationCpuUtilizationThreshold <= 0) { LOG.info("NodeManager oversubscription is disabled because the CPU " + - "utilization threshold is no larger than zero."); + "overallocation threshold is no larger than zero."); return; } - float preemptionThreshold = conf.getFloat( - YarnConfiguration.NM_OVERALLOCATION_PREEMPTION_THRESHOLD, - YarnConfiguration.DEFAULT_NM_OVERALLOCATION_PREEMPTION_THRESHOLD); - if (preemptionThreshold <= overAllocationCpuUtilizationThreshold) { - LOG.info("NodeManager oversubscription is disabled because preemption" + - "threshold is no larger than the cpu utilization threshold."); + float cpuPreemptionThreshold = conf.getFloat( + YarnConfiguration.NM_OVERALLOCATION_CPU_PREEMPTION_THRESHOLD, + YarnConfiguration. + DEFAULT_NM_OVERALLOCATION_CPU_PREEMPTION_THRESHOLD); + if (cpuPreemptionThreshold <= overAllocationCpuUtilizationThreshold) { + LOG.info("NodeManager oversubscription is disabled because the cpu " + + " preemption threshold is no larger than the cpu overallocation" + + " threshold."); return; } - if (preemptionThreshold <= overAllocationMemoryUtilizationThreshold) { - LOG.info("NodeManager oversubscription is disabled because preemption" + - "threshold is no larger than the memory utilization threshold."); + + float memoryPreemptionThreshold = conf.getFloat( + YarnConfiguration.NM_OVERALLOCATION_MEMORY_PREEMPTION_THRESHOLD, + YarnConfiguration. + DEFAULT_NM_OVERALLOCATION_MEMORY_PREEMPTION_THRESHOLD); + if (memoryPreemptionThreshold <= overAllocationMemoryUtilizationThreshold) { + LOG.info("NodeManager oversubscription is disabled because the memory" + + " preemption threshold is no larger than the memory overallocation" + + " threshold."); return; } + this.overAlloctionPreemptionCpuCount = conf.getInt( + YarnConfiguration.NM_OVERALLOCATION_PREEMPTION_CPU_COUNT, + YarnConfiguration.DEFAULT_NM_OVERALLOCATION_PREEMPTION_CPU_COUNT); + ResourceThresholds resourceThresholds = ResourceThresholds.newInstance( overAllocationCpuUtilizationThreshold, overAllocationMemoryUtilizationThreshold); ((NodeManager.NMContext) context).setOverAllocationInfo( OverAllocationInfo.newInstance(resourceThresholds)); - this.overAllocationPreemptionThresholds = - ResourceThresholds.newInstance(preemptionThreshold); + this.overAllocationPreemptionThresholds = ResourceThresholds.newInstance( + cpuPreemptionThreshold, memoryPreemptionThreshold); LOG.info("NodeManager oversubscription enabled with overallocation " + "thresholds (memory:" + overAllocationMemoryUtilizationThreshold + ", CPU:" + overAllocationCpuUtilizationThreshold + ") and preemption" + - " threshold: " + preemptionThreshold); + " threshold (memory:" + memoryPreemptionThreshold + ", CPU:" + + cpuPreemptionThreshold + ")"); } private boolean isResourceCalculatorAvailable() {