diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 03fa1ce..2be9523 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -1611,10 +1611,33 @@ public static boolean isAclEnabled(Configuration conf) {
public static final String NM_OVERALLOCATION_MEMORY_UTILIZATION_THRESHOLD =
NM_PREFIX + "overallocation.memory-utilization-threshold";
- public static final String NM_OVERALLOCATION_PREEMPTION_THRESHOLD =
- NM_PREFIX + "overallocation.preemption-threshold";
- public static final float DEFAULT_NM_OVERALLOCATION_PREEMPTION_THRESHOLD
- = 0.96f;
+ /**
+ * The CPU utilization threshold, if went beyond for a few times in a row,
+ * OPPORTUNISTIC containers started due to overallocation should start
+ * getting preempted.
+ */
+ public static final String NM_OVERALLOCATION_CPU_PREEMPTION_THRESHOLD =
+ NM_PREFIX + "overallocation.preemption-threshold.cpu";
+ public static final float
+ DEFAULT_NM_OVERALLOCATION_CPU_PREEMPTION_THRESHOLD = 0.99f;
+
+ /**
+ * The number of times that CPU utilization must go over the CPU preemption
+ * threshold consecutively before preemption starts to kick in.
+ */
+ public static final String NM_OVERALLOCATION_PREEMPTION_CPU_COUNT =
+ NM_PREFIX + "overallocation.preemption-threshold-count.cpu";
+ public static final int DEFAULT_NM_OVERALLOCATION_PREEMPTION_CPU_COUNT = 4;
+
+
+ /**
+ * The memory utilization threshold beyond which OPPORTUNISTIC containers
+ * started due to overallocation should start getting preempted.
+ */
+ public static final String NM_OVERALLOCATION_MEMORY_PREEMPTION_THRESHOLD =
+ NM_PREFIX + "overallocation.preemption-threshold.memory";
+ public static final float
+ DEFAULT_NM_OVERALLOCATION_MEMORY_PREEMPTION_THRESHOLD = 0.95f;
/**
* Interval of time the linux container executor should try cleaning up
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 42166a9..837e9f3 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -1606,11 +1606,37 @@
When a node is over-allocated to improve utilization by
- running OPPORTUNISTIC containers, this config captures the utilization
- beyond which OPPORTUNISTIC containers should start getting preempted.
+ running OPPORTUNISTIC containers, this config captures the CPU
+ utilization beyond which OPPORTUNISTIC containers should start getting
+ preempted. This is used in combination with
+ yarn.nodemanager.overallocation.preemption-threshold-count.cpu, that is,
+ only when the CPU utilization goes over this threshold consecutively for
+ a few times will preemption kicks in.
- yarn.nodemanager.overallocation.preemption-threshold
- 0.96
+ yarn.nodemanager.overallocation.preemption-threshold.cpu
+ 0.99
+
+
+
+ When a node is over-allocated to improve utilization by
+ running OPPORTUNISTIC containers, this config captures the number of
+ times that CPU utilization has to go above
+ ${yarn.nodemanager.overallocation.preemption-threshold.cpu}
+ consecutively for NM to start preempting OPPORTUNISTIC containers
+ started due to overallocation.
+
+ yarn.nodemanager.overallocation.preemption-threshold-count.cpu
+ 4
+
+
+
+ When a node is over-allocated to improve utilization by
+ running OPPORTUNISTIC containers, this config captures the CPU
+ utilization beyond which OPPORTUNISTIC containers should start getting
+ preempted.
+
+ yarn.nodemanager.overallocation.preemption-threshold.memory
+ 0.95
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
index a7cbc5b..1e41803 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
@@ -102,6 +102,7 @@
private ResourceUtilization containersUtilization;
private ResourceThresholds overAllocationPreemptionThresholds;
+ private int overAlloctionPreemptionCpuCount = -1;
private volatile boolean stopped = false;
@@ -236,7 +237,7 @@ private void initializeOverAllocation(Configuration conf) {
YarnConfiguration.MAX_NM_OVERALLOCATION_THRESHOLD);
if (overAllocationMemoryUtilizationThreshold <= 0) {
LOG.info("NodeManager oversubscription is disabled because the memory " +
- "utilization threshold is no larger than zero.");
+ "overallocation threshold is no larger than zero.");
return;
}
@@ -248,36 +249,49 @@ private void initializeOverAllocation(Configuration conf) {
YarnConfiguration.MAX_NM_OVERALLOCATION_THRESHOLD);
if (overAllocationCpuUtilizationThreshold <= 0) {
LOG.info("NodeManager oversubscription is disabled because the CPU " +
- "utilization threshold is no larger than zero.");
+ "overallocation threshold is no larger than zero.");
return;
}
- float preemptionThreshold = conf.getFloat(
- YarnConfiguration.NM_OVERALLOCATION_PREEMPTION_THRESHOLD,
- YarnConfiguration.DEFAULT_NM_OVERALLOCATION_PREEMPTION_THRESHOLD);
- if (preemptionThreshold <= overAllocationCpuUtilizationThreshold) {
- LOG.info("NodeManager oversubscription is disabled because preemption" +
- "threshold is no larger than the cpu utilization threshold.");
+ float cpuPreemptionThreshold = conf.getFloat(
+ YarnConfiguration.NM_OVERALLOCATION_CPU_PREEMPTION_THRESHOLD,
+ YarnConfiguration.
+ DEFAULT_NM_OVERALLOCATION_CPU_PREEMPTION_THRESHOLD);
+ if (cpuPreemptionThreshold <= overAllocationCpuUtilizationThreshold) {
+ LOG.info("NodeManager oversubscription is disabled because the cpu " +
+ " preemption threshold is no larger than the cpu overallocation" +
+ " threshold.");
return;
}
- if (preemptionThreshold <= overAllocationMemoryUtilizationThreshold) {
- LOG.info("NodeManager oversubscription is disabled because preemption" +
- "threshold is no larger than the memory utilization threshold.");
+
+ float memoryPreemptionThreshold = conf.getFloat(
+ YarnConfiguration.NM_OVERALLOCATION_MEMORY_PREEMPTION_THRESHOLD,
+ YarnConfiguration.
+ DEFAULT_NM_OVERALLOCATION_MEMORY_PREEMPTION_THRESHOLD);
+ if (memoryPreemptionThreshold <= overAllocationMemoryUtilizationThreshold) {
+ LOG.info("NodeManager oversubscription is disabled because the memory" +
+ " preemption threshold is no larger than the memory overallocation" +
+ " threshold.");
return;
}
+ this.overAlloctionPreemptionCpuCount = conf.getInt(
+ YarnConfiguration.NM_OVERALLOCATION_PREEMPTION_CPU_COUNT,
+ YarnConfiguration.DEFAULT_NM_OVERALLOCATION_PREEMPTION_CPU_COUNT);
+
ResourceThresholds resourceThresholds = ResourceThresholds.newInstance(
overAllocationCpuUtilizationThreshold,
overAllocationMemoryUtilizationThreshold);
((NodeManager.NMContext) context).setOverAllocationInfo(
OverAllocationInfo.newInstance(resourceThresholds));
- this.overAllocationPreemptionThresholds =
- ResourceThresholds.newInstance(preemptionThreshold);
+ this.overAllocationPreemptionThresholds = ResourceThresholds.newInstance(
+ cpuPreemptionThreshold, memoryPreemptionThreshold);
LOG.info("NodeManager oversubscription enabled with overallocation " +
"thresholds (memory:" + overAllocationMemoryUtilizationThreshold +
", CPU:" + overAllocationCpuUtilizationThreshold + ") and preemption" +
- " threshold: " + preemptionThreshold);
+ " threshold (memory:" + memoryPreemptionThreshold + ", CPU:" +
+ cpuPreemptionThreshold + ")");
}
private boolean isResourceCalculatorAvailable() {