diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 03fa1ce..19db29b 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -1611,10 +1611,37 @@ public static boolean isAclEnabled(Configuration conf) { public static final String NM_OVERALLOCATION_MEMORY_UTILIZATION_THRESHOLD = NM_PREFIX + "overallocation.memory-utilization-threshold"; - public static final String NM_OVERALLOCATION_PREEMPTION_THRESHOLD = - NM_PREFIX + "overallocation.preemption-threshold"; - public static final float DEFAULT_NM_OVERALLOCATION_PREEMPTION_THRESHOLD - = 0.96f; + /** + * The maximum value of preemption threshold for all resource types + * beyond which OPPORTUNISTIC containers start getting preempted. + * This must be set larger than MAX_NM_OVERALLOCATION_THRESHOLD. + */ + @Private + public static final float MAX_NM_PREEMPTION_THRESHOLD = 0.97f; + + /** + * The general utilization threshold beyond which OPPORTUNISTIC containers + * started due to overallocation should start getting preempted. + */ + public static final String NM_OVERALLOCATION_GENERAL_PREEMPTION_THRESHOLD = + NM_PREFIX + "overallocation.general-preemption-threshold"; + public static final float + DEFAULT_NM_OVERALLOCATION_GENERAL_PREEMPTION_THRESHOLD + = MAX_NM_PREEMPTION_THRESHOLD; + + /** + * The CPU utilization threshold beyond which OPPORTUNISTIC containers + * started due to overallocation should start getting preempted. + */ + public static final String NM_OVERALLOCATION_CPU_PREEMPTION_THRESHOLD = + NM_PREFIX + "overallocation.cpu-preemption-threshold"; + + /** + * The memory utilization threshold beyond which OPPORTUNISTIC containers + * started due to overallocation should start getting preempted. + */ + public static final String NM_OVERALLOCATION_MEMORY_PREEMPTION_THRESHOLD = + NM_PREFIX + "overallocation.memory-preemption-threshold"; /** * Interval of time the linux container executor should try cleaning up diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 42166a9..5fb94ce 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -1605,12 +1605,36 @@ + The general utilization threshold, if no resource- + type-specific setting is configured, beyond which OPPORTUNISTIC containers + should start getting preempted when a node is over-allocated to improve + utilization by running OPPORTUNISTIC containers. Note for each + resource type, it can be overridden by resource-type-specific setting: + yarn.nodemanager.overallocation.cpu-preemption-threshold for CPU + yarn.nodemanager.overallocation.memory-preemption-threshold for memory + + yarn.nodemanager.overallocation.general-preemption-threshold + 0.97 + + + + When a node is over-allocated to improve utilization by + running OPPORTUNISTIC containers, this config captures the CPU + utilization beyond which OPPORTUNISTIC containers should start getting + preempted. + + yarn.nodemanager.overallocation.cpu-preemption-threshold + ${yarn.nodemanager.overallocation.general-preemption-threshold} + + + When a node is over-allocated to improve utilization by - running OPPORTUNISTIC containers, this config captures the utilization - beyond which OPPORTUNISTIC containers should start getting preempted. + running OPPORTUNISTIC containers, this config captures the CPU + utilization beyond which OPPORTUNISTIC containers should start getting + preempted. - yarn.nodemanager.overallocation.preemption-threshold - 0.96 + yarn.nodemanager.overallocation.memory-preemption-threshold + ${yarn.nodemanager.overallocation.general-preemption-threshold} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java index a7cbc5b..1ce71ae 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java @@ -236,7 +236,7 @@ private void initializeOverAllocation(Configuration conf) { YarnConfiguration.MAX_NM_OVERALLOCATION_THRESHOLD); if (overAllocationMemoryUtilizationThreshold <= 0) { LOG.info("NodeManager oversubscription is disabled because the memory " + - "utilization threshold is no larger than zero."); + "overallocation threshold is no larger than zero."); return; } @@ -248,21 +248,35 @@ private void initializeOverAllocation(Configuration conf) { YarnConfiguration.MAX_NM_OVERALLOCATION_THRESHOLD); if (overAllocationCpuUtilizationThreshold <= 0) { LOG.info("NodeManager oversubscription is disabled because the CPU " + - "utilization threshold is no larger than zero."); + "overallocation threshold is no larger than zero."); return; } - float preemptionThreshold = conf.getFloat( - YarnConfiguration.NM_OVERALLOCATION_PREEMPTION_THRESHOLD, - YarnConfiguration.DEFAULT_NM_OVERALLOCATION_PREEMPTION_THRESHOLD); - if (preemptionThreshold <= overAllocationCpuUtilizationThreshold) { - LOG.info("NodeManager oversubscription is disabled because preemption" + - "threshold is no larger than the cpu utilization threshold."); + float generalPreemptionThreshold = conf.getFloat( + YarnConfiguration.NM_OVERALLOCATION_GENERAL_PREEMPTION_THRESHOLD, + YarnConfiguration.DEFAULT_NM_OVERALLOCATION_GENERAL_PREEMPTION_THRESHOLD); + + float cpuPreemptionThreshold = conf.getFloat( + YarnConfiguration.NM_OVERALLOCATION_CPU_PREEMPTION_THRESHOLD, + YarnConfiguration.DEFAULT_NM_OVERALLOCATION_GENERAL_PREEMPTION_THRESHOLD); + cpuPreemptionThreshold = Math.min(cpuPreemptionThreshold, + YarnConfiguration.MAX_NM_PREEMPTION_THRESHOLD); + if (cpuPreemptionThreshold <= overAllocationCpuUtilizationThreshold) { + LOG.info("NodeManager oversubscription is disabled because the cpu " + + " preemption threshold is no larger than the cpu overallocation" + + " threshold."); return; } - if (preemptionThreshold <= overAllocationMemoryUtilizationThreshold) { - LOG.info("NodeManager oversubscription is disabled because preemption" + - "threshold is no larger than the memory utilization threshold."); + + float memoryPreemptionThreshold = conf.getFloat( + YarnConfiguration.NM_OVERALLOCATION_MEMORY_PREEMPTION_THRESHOLD, + YarnConfiguration.DEFAULT_NM_OVERALLOCATION_GENERAL_PREEMPTION_THRESHOLD); + memoryPreemptionThreshold = Math.min(memoryPreemptionThreshold, + YarnConfiguration.MAX_NM_PREEMPTION_THRESHOLD); + if (memoryPreemptionThreshold <= overAllocationMemoryUtilizationThreshold) { + LOG.info("NodeManager oversubscription is disabled because the memory" + + " preemption threshold is no larger than the memory overallocation" + + " threshold."); return; } @@ -272,12 +286,13 @@ private void initializeOverAllocation(Configuration conf) { ((NodeManager.NMContext) context).setOverAllocationInfo( OverAllocationInfo.newInstance(resourceThresholds)); this.overAllocationPreemptionThresholds = - ResourceThresholds.newInstance(preemptionThreshold); + ResourceThresholds.newInstance(generalPreemptionThreshold); LOG.info("NodeManager oversubscription enabled with overallocation " + "thresholds (memory:" + overAllocationMemoryUtilizationThreshold + ", CPU:" + overAllocationCpuUtilizationThreshold + ") and preemption" + - " threshold: " + preemptionThreshold); + " threshold (memory:" + memoryPreemptionThreshold + ", CPU:" + + cpuPreemptionThreshold + ")"); } private boolean isResourceCalculatorAvailable() {